2024-11-20 05:14:09,610 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 05:14:09,621 main DEBUG Took 0.009456 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 05:14:09,622 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 05:14:09,622 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 05:14:09,623 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 05:14:09,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,631 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 05:14:09,642 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,643 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,644 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,644 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,646 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,646 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,647 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,648 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,649 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,649 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,650 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,651 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,651 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,652 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 05:14:09,653 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,653 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 05:14:09,654 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 05:14:09,655 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 05:14:09,657 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 05:14:09,657 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 05:14:09,658 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 05:14:09,659 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 05:14:09,666 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 05:14:09,668 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 05:14:09,670 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 05:14:09,670 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 05:14:09,670 main DEBUG createAppenders(={Console}) 2024-11-20 05:14:09,671 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 05:14:09,671 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 05:14:09,671 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 05:14:09,672 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 05:14:09,672 main DEBUG OutputStream closed 2024-11-20 05:14:09,672 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 05:14:09,673 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 05:14:09,673 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 05:14:09,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 05:14:09,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 05:14:09,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 05:14:09,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 05:14:09,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 05:14:09,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 05:14:09,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 05:14:09,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 05:14:09,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 05:14:09,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 05:14:09,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 05:14:09,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 05:14:09,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 05:14:09,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 05:14:09,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 05:14:09,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 05:14:09,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 05:14:09,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 05:14:09,749 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 05:14:09,750 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 05:14:09,750 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 05:14:09,751 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T05:14:09,971 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca 2024-11-20 05:14:09,974 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 05:14:09,974 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T05:14:09,981 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-20T05:14:10,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T05:14:10,020 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1, deleteOnExit=true 2024-11-20T05:14:10,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T05:14:10,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/test.cache.data in system properties and HBase conf 2024-11-20T05:14:10,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T05:14:10,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/hadoop.log.dir in system properties and HBase conf 2024-11-20T05:14:10,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T05:14:10,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T05:14:10,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T05:14:10,118 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T05:14:10,202 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T05:14:10,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T05:14:10,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T05:14:10,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T05:14:10,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T05:14:10,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T05:14:10,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T05:14:10,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T05:14:10,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T05:14:10,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T05:14:10,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/nfs.dump.dir in system properties and HBase conf 2024-11-20T05:14:10,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/java.io.tmpdir in system properties and HBase conf 2024-11-20T05:14:10,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T05:14:10,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T05:14:10,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T05:14:11,262 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T05:14:11,331 INFO [Time-limited test {}] log.Log(170): Logging initialized @2399ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T05:14:11,395 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T05:14:11,451 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T05:14:11,474 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T05:14:11,474 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T05:14:11,475 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T05:14:11,487 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T05:14:11,490 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/hadoop.log.dir/,AVAILABLE} 2024-11-20T05:14:11,491 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T05:14:11,690 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/java.io.tmpdir/jetty-localhost-38427-hadoop-hdfs-3_4_1-tests_jar-_-any-6532060460560360724/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T05:14:11,696 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:38427} 2024-11-20T05:14:11,696 INFO [Time-limited test {}] server.Server(415): Started @2765ms 2024-11-20T05:14:12,335 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T05:14:12,344 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T05:14:12,345 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T05:14:12,345 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T05:14:12,346 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T05:14:12,347 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/hadoop.log.dir/,AVAILABLE} 2024-11-20T05:14:12,347 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T05:14:12,460 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/java.io.tmpdir/jetty-localhost-38263-hadoop-hdfs-3_4_1-tests_jar-_-any-10574756653482200770/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T05:14:12,461 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:38263} 2024-11-20T05:14:12,461 INFO [Time-limited test {}] server.Server(415): Started @3530ms 2024-11-20T05:14:12,512 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T05:14:13,639 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/data/data2/current/BP-1659978148-172.17.0.3-1732079650746/current, will proceed with Du for space computation calculation, 2024-11-20T05:14:13,639 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/data/data1/current/BP-1659978148-172.17.0.3-1732079650746/current, will proceed with Du for space computation calculation, 2024-11-20T05:14:13,669 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T05:14:13,707 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x54a31731df4605b with lease ID 0x7e7a8944ee6e11be: Processing first storage report for DS-1d9f8dc9-dffd-4de9-8f84-9e820d719358 from datanode DatanodeRegistration(127.0.0.1:36259, datanodeUuid=534a6e25-a7f9-42cd-a80a-545dcd92dfad, infoPort=43431, infoSecurePort=0, ipcPort=41865, storageInfo=lv=-57;cid=testClusterID;nsid=1537854728;c=1732079650746) 2024-11-20T05:14:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x54a31731df4605b with lease ID 0x7e7a8944ee6e11be: from storage DS-1d9f8dc9-dffd-4de9-8f84-9e820d719358 node DatanodeRegistration(127.0.0.1:36259, datanodeUuid=534a6e25-a7f9-42cd-a80a-545dcd92dfad, infoPort=43431, infoSecurePort=0, ipcPort=41865, storageInfo=lv=-57;cid=testClusterID;nsid=1537854728;c=1732079650746), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T05:14:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x54a31731df4605b with lease ID 0x7e7a8944ee6e11be: Processing first storage report for DS-03f3f5e8-8575-4b68-badd-d37143dd4915 from datanode DatanodeRegistration(127.0.0.1:36259, datanodeUuid=534a6e25-a7f9-42cd-a80a-545dcd92dfad, infoPort=43431, infoSecurePort=0, ipcPort=41865, storageInfo=lv=-57;cid=testClusterID;nsid=1537854728;c=1732079650746) 2024-11-20T05:14:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x54a31731df4605b with lease ID 0x7e7a8944ee6e11be: from storage DS-03f3f5e8-8575-4b68-badd-d37143dd4915 node DatanodeRegistration(127.0.0.1:36259, datanodeUuid=534a6e25-a7f9-42cd-a80a-545dcd92dfad, infoPort=43431, infoSecurePort=0, ipcPort=41865, storageInfo=lv=-57;cid=testClusterID;nsid=1537854728;c=1732079650746), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T05:14:13,793 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca 2024-11-20T05:14:13,857 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/zookeeper_0, clientPort=60391, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T05:14:13,866 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60391 2024-11-20T05:14:13,876 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T05:14:13,878 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T05:14:14,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741825_1001 (size=7) 2024-11-20T05:14:14,488 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 with version=8 2024-11-20T05:14:14,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/hbase-staging 2024-11-20T05:14:14,573 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T05:14:14,817 INFO [Time-limited test {}] client.ConnectionUtils(128): master/51d5d2011a1c:0 server-side Connection retries=45 2024-11-20T05:14:14,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T05:14:14,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T05:14:14,830 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T05:14:14,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T05:14:14,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T05:14:14,951 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T05:14:15,002 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T05:14:15,011 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T05:14:15,014 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T05:14:15,036 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4683 (auto-detected) 2024-11-20T05:14:15,037 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-20T05:14:15,054 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37927 2024-11-20T05:14:15,075 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37927 connecting to ZooKeeper ensemble=127.0.0.1:60391 2024-11-20T05:14:15,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379270x0, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T05:14:15,190 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37927-0x10156aa0f7c0000 connected 2024-11-20T05:14:15,283 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T05:14:15,290 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T05:14:15,300 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T05:14:15,304 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673, hbase.cluster.distributed=false 2024-11-20T05:14:15,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T05:14:15,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37927 2024-11-20T05:14:15,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37927 2024-11-20T05:14:15,337 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37927 2024-11-20T05:14:15,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37927 2024-11-20T05:14:15,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37927 2024-11-20T05:14:15,432 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51d5d2011a1c:0 server-side Connection retries=45 2024-11-20T05:14:15,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T05:14:15,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T05:14:15,434 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T05:14:15,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T05:14:15,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T05:14:15,437 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T05:14:15,440 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T05:14:15,441 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44753 2024-11-20T05:14:15,443 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44753 connecting to ZooKeeper ensemble=127.0.0.1:60391 2024-11-20T05:14:15,444 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T05:14:15,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T05:14:15,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447530x0, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T05:14:15,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447530x0, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T05:14:15,459 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44753-0x10156aa0f7c0001 connected 2024-11-20T05:14:15,463 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T05:14:15,471 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T05:14:15,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T05:14:15,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T05:14:15,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44753 2024-11-20T05:14:15,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44753 2024-11-20T05:14:15,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44753 2024-11-20T05:14:15,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44753 2024-11-20T05:14:15,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44753 2024-11-20T05:14:15,497 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;51d5d2011a1c:37927 2024-11-20T05:14:15,500 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T05:14:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T05:14:15,514 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:15,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:15,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T05:14:15,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:15,544 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T05:14:15,545 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/51d5d2011a1c,37927,1732079654666 from backup master directory 2024-11-20T05:14:15,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:15,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T05:14:15,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T05:14:15,554 WARN [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T05:14:15,554 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:15,556 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T05:14:15,558 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T05:14:15,611 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/hbase.id] with ID: 4b9515f8-070c-401f-827a-a94aa4386eb6 2024-11-20T05:14:15,611 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/.tmp/hbase.id 2024-11-20T05:14:15,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741826_1002 (size=42) 2024-11-20T05:14:15,624 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/.tmp/hbase.id]:[hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/hbase.id] 2024-11-20T05:14:15,671 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T05:14:15,676 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T05:14:15,695 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-20T05:14:15,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:15,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:15,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741827_1003 (size=196) 2024-11-20T05:14:15,742 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:14:15,745 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T05:14:15,760 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:15,764 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T05:14:15,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741828_1004 (size=1189) 2024-11-20T05:14:16,217 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store 2024-11-20T05:14:16,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741829_1005 (size=34) 2024-11-20T05:14:16,649 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T05:14:16,652 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:16,653 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T05:14:16,653 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:14:16,654 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:14:16,655 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T05:14:16,655 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:14:16,656 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:14:16,657 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732079656653Disabling compacts and flushes for region at 1732079656653Disabling writes for close at 1732079656655 (+2 ms)Writing region close event to WAL at 1732079656655Closed at 1732079656655 2024-11-20T05:14:16,659 WARN [master/51d5d2011a1c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/.initializing 2024-11-20T05:14:16,660 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/WALs/51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:16,668 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T05:14:16,683 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51d5d2011a1c%2C37927%2C1732079654666, suffix=, logDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/WALs/51d5d2011a1c,37927,1732079654666, archiveDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/oldWALs, maxLogs=10 2024-11-20T05:14:16,704 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/WALs/51d5d2011a1c,37927,1732079654666/51d5d2011a1c%2C37927%2C1732079654666.1732079656688, exclude list is [], retry=0 2024-11-20T05:14:16,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36259,DS-1d9f8dc9-dffd-4de9-8f84-9e820d719358,DISK] 2024-11-20T05:14:16,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T05:14:16,752 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/WALs/51d5d2011a1c,37927,1732079654666/51d5d2011a1c%2C37927%2C1732079654666.1732079656688 2024-11-20T05:14:16,753 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43431:43431)] 2024-11-20T05:14:16,753 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:14:16,754 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:16,756 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,757 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T05:14:16,817 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:16,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:16,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T05:14:16,824 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:16,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:16,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T05:14:16,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:16,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:16,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T05:14:16,832 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:16,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:16,833 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,836 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,838 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,842 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,843 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,847 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T05:14:16,850 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T05:14:16,855 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:14:16,856 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60517374, jitterRate=-0.09822085499763489}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T05:14:16,862 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732079656768Initializing all the Stores at 1732079656770 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732079656770Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079656771 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079656771Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079656771Cleaning up temporary data from old regions at 1732079656843 (+72 ms)Region opened successfully at 1732079656862 (+19 ms) 2024-11-20T05:14:16,863 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T05:14:16,894 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77061cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51d5d2011a1c/172.17.0.3:0 2024-11-20T05:14:16,921 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T05:14:16,930 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T05:14:16,930 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T05:14:16,933 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T05:14:16,934 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T05:14:16,938 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T05:14:16,938 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T05:14:16,961 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T05:14:16,969 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T05:14:17,046 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T05:14:17,050 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T05:14:17,053 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T05:14:17,058 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T05:14:17,061 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T05:14:17,065 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T05:14:17,077 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T05:14:17,080 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T05:14:17,090 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T05:14:17,110 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T05:14:17,121 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T05:14:17,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T05:14:17,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T05:14:17,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:17,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:17,136 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=51d5d2011a1c,37927,1732079654666, sessionid=0x10156aa0f7c0000, setting cluster-up flag (Was=false) 2024-11-20T05:14:17,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:17,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:17,195 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T05:14:17,198 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:17,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:17,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:17,248 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T05:14:17,252 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:17,262 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T05:14:17,288 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(746): ClusterId : 4b9515f8-070c-401f-827a-a94aa4386eb6 2024-11-20T05:14:17,290 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T05:14:17,303 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T05:14:17,303 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T05:14:17,312 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T05:14:17,313 DEBUG [RS:0;51d5d2011a1c:44753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53192a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51d5d2011a1c/172.17.0.3:0 2024-11-20T05:14:17,329 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T05:14:17,329 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;51d5d2011a1c:44753 2024-11-20T05:14:17,332 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T05:14:17,332 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T05:14:17,332 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T05:14:17,335 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(2659): reportForDuty to master=51d5d2011a1c,37927,1732079654666 with port=44753, startcode=1732079655401 2024-11-20T05:14:17,337 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T05:14:17,343 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T05:14:17,346 DEBUG [RS:0;51d5d2011a1c:44753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T05:14:17,348 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 51d5d2011a1c,37927,1732079654666 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T05:14:17,354 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/51d5d2011a1c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T05:14:17,354 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/51d5d2011a1c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T05:14:17,354 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/51d5d2011a1c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T05:14:17,354 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/51d5d2011a1c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T05:14:17,354 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/51d5d2011a1c:0, corePoolSize=10, maxPoolSize=10 2024-11-20T05:14:17,355 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,355 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/51d5d2011a1c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T05:14:17,355 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,357 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732079687357 2024-11-20T05:14:17,359 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T05:14:17,360 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T05:14:17,361 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T05:14:17,361 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T05:14:17,363 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T05:14:17,364 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T05:14:17,364 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T05:14:17,364 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T05:14:17,367 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,369 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:17,369 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T05:14:17,372 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T05:14:17,373 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T05:14:17,374 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T05:14:17,378 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T05:14:17,381 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T05:14:17,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741831_1007 (size=1321) 2024-11-20T05:14:17,384 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41011, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T05:14:17,385 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T05:14:17,386 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:14:17,391 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/51d5d2011a1c:0:becomeActiveMaster-HFileCleaner.large.0-1732079657384,5,FailOnTimeoutGroup] 2024-11-20T05:14:17,392 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/51d5d2011a1c:0:becomeActiveMaster-HFileCleaner.small.0-1732079657391,5,FailOnTimeoutGroup] 2024-11-20T05:14:17,392 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,392 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T05:14:17,394 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,394 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,395 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37927 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:17,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741832_1008 (size=32) 2024-11-20T05:14:17,421 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T05:14:17,421 WARN [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T05:14:17,523 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(2659): reportForDuty to master=51d5d2011a1c,37927,1732079654666 with port=44753, startcode=1732079655401 2024-11-20T05:14:17,526 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37927 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:17,528 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37927 {}] master.ServerManager(517): Registering regionserver=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:17,537 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:14:17,537 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39657 2024-11-20T05:14:17,537 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T05:14:17,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T05:14:17,551 DEBUG [RS:0;51d5d2011a1c:44753 {}] zookeeper.ZKUtil(111): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:17,552 WARN [RS:0;51d5d2011a1c:44753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T05:14:17,552 INFO [RS:0;51d5d2011a1c:44753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T05:14:17,552 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:17,554 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51d5d2011a1c,44753,1732079655401] 2024-11-20T05:14:17,581 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T05:14:17,597 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T05:14:17,601 INFO [RS:0;51d5d2011a1c:44753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T05:14:17,601 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,602 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T05:14:17,607 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T05:14:17,608 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,608 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,608 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,609 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,609 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,609 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,609 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51d5d2011a1c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T05:14:17,609 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,609 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,609 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,610 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,610 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,610 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51d5d2011a1c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T05:14:17,610 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51d5d2011a1c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T05:14:17,610 DEBUG [RS:0;51d5d2011a1c:44753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T05:14:17,611 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,612 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,612 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,612 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,612 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,612 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,44753,1732079655401-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T05:14:17,630 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T05:14:17,632 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,44753,1732079655401-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,632 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,632 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.Replication(171): 51d5d2011a1c,44753,1732079655401 started 2024-11-20T05:14:17,648 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:17,649 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1482): Serving as 51d5d2011a1c,44753,1732079655401, RpcServer on 51d5d2011a1c/172.17.0.3:44753, sessionid=0x10156aa0f7c0001 2024-11-20T05:14:17,649 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T05:14:17,650 DEBUG [RS:0;51d5d2011a1c:44753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:17,650 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51d5d2011a1c,44753,1732079655401' 2024-11-20T05:14:17,650 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T05:14:17,651 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T05:14:17,651 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T05:14:17,652 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T05:14:17,652 DEBUG [RS:0;51d5d2011a1c:44753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:17,652 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51d5d2011a1c,44753,1732079655401' 2024-11-20T05:14:17,652 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T05:14:17,653 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T05:14:17,653 DEBUG [RS:0;51d5d2011a1c:44753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T05:14:17,653 INFO [RS:0;51d5d2011a1c:44753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T05:14:17,653 INFO [RS:0;51d5d2011a1c:44753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T05:14:17,761 INFO [RS:0;51d5d2011a1c:44753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T05:14:17,765 INFO [RS:0;51d5d2011a1c:44753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51d5d2011a1c%2C44753%2C1732079655401, suffix=, logDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401, archiveDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/oldWALs, maxLogs=32 2024-11-20T05:14:17,779 DEBUG [RS:0;51d5d2011a1c:44753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401/51d5d2011a1c%2C44753%2C1732079655401.1732079657767, exclude list is [], retry=0 2024-11-20T05:14:17,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36259,DS-1d9f8dc9-dffd-4de9-8f84-9e820d719358,DISK] 2024-11-20T05:14:17,787 INFO [RS:0;51d5d2011a1c:44753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401/51d5d2011a1c%2C44753%2C1732079655401.1732079657767 2024-11-20T05:14:17,787 DEBUG [RS:0;51d5d2011a1c:44753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43431:43431)] 2024-11-20T05:14:17,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:17,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T05:14:17,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T05:14:17,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:17,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:17,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T05:14:17,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T05:14:17,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:17,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:17,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T05:14:17,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T05:14:17,819 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:17,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:17,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T05:14:17,823 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T05:14:17,823 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:17,824 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:17,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T05:14:17,825 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740 2024-11-20T05:14:17,826 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740 2024-11-20T05:14:17,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T05:14:17,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T05:14:17,830 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:14:17,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T05:14:17,837 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:14:17,838 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67558357, jitterRate=0.0066979676485061646}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:14:17,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732079657806Initializing all the Stores at 1732079657808 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732079657808Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732079657809 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079657809Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732079657809Cleaning up temporary data from old regions at 1732079657829 (+20 ms)Region opened successfully at 1732079657840 (+11 ms) 2024-11-20T05:14:17,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T05:14:17,841 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T05:14:17,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T05:14:17,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T05:14:17,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T05:14:17,843 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T05:14:17,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732079657840Disabling compacts and flushes for region at 1732079657840Disabling writes for close at 1732079657841 (+1 ms)Writing region close event to WAL at 1732079657842 (+1 ms)Closed at 1732079657843 (+1 ms) 2024-11-20T05:14:17,846 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T05:14:17,847 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T05:14:17,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T05:14:17,859 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T05:14:17,861 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T05:14:18,016 DEBUG [51d5d2011a1c:37927 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T05:14:18,029 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:18,035 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51d5d2011a1c,44753,1732079655401, state=OPENING 2024-11-20T05:14:18,058 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T05:14:18,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:18,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:14:18,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T05:14:18,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T05:14:18,074 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T05:14:18,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:14:18,254 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T05:14:18,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45509, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T05:14:18,267 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T05:14:18,267 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T05:14:18,268 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T05:14:18,271 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51d5d2011a1c%2C44753%2C1732079655401.meta, suffix=.meta, logDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401, archiveDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/oldWALs, maxLogs=32 2024-11-20T05:14:18,286 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401/51d5d2011a1c%2C44753%2C1732079655401.meta.1732079658273.meta, exclude list is [], retry=0 2024-11-20T05:14:18,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36259,DS-1d9f8dc9-dffd-4de9-8f84-9e820d719358,DISK] 2024-11-20T05:14:18,293 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401/51d5d2011a1c%2C44753%2C1732079655401.meta.1732079658273.meta 2024-11-20T05:14:18,293 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43431:43431)] 2024-11-20T05:14:18,293 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:14:18,295 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T05:14:18,297 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T05:14:18,302 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T05:14:18,305 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T05:14:18,306 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:18,306 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T05:14:18,306 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T05:14:18,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T05:14:18,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T05:14:18,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:18,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:18,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T05:14:18,314 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T05:14:18,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:18,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:18,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T05:14:18,316 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T05:14:18,316 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:18,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:18,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T05:14:18,319 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T05:14:18,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:18,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T05:14:18,320 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T05:14:18,322 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740 2024-11-20T05:14:18,324 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740 2024-11-20T05:14:18,327 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T05:14:18,327 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T05:14:18,328 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:14:18,330 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T05:14:18,332 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61228009, jitterRate=-0.08763156831264496}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:14:18,332 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T05:14:18,333 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732079658307Writing region info on filesystem at 1732079658307Initializing all the Stores at 1732079658309 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732079658309Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732079658309Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079658309Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732079658309Cleaning up temporary data from old regions at 1732079658327 (+18 ms)Running coprocessor post-open hooks at 1732079658332 (+5 ms)Region opened successfully at 1732079658333 (+1 ms) 2024-11-20T05:14:18,340 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732079658244 2024-11-20T05:14:18,350 DEBUG [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T05:14:18,351 INFO [RS_OPEN_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T05:14:18,352 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:18,355 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51d5d2011a1c,44753,1732079655401, state=OPEN 2024-11-20T05:14:18,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T05:14:18,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T05:14:18,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T05:14:18,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T05:14:18,393 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:18,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T05:14:18,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51d5d2011a1c,44753,1732079655401 in 316 msec 2024-11-20T05:14:18,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T05:14:18,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 551 msec 2024-11-20T05:14:18,411 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T05:14:18,411 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T05:14:18,428 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:18,430 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:18,447 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:18,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53651, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:18,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1770 sec 2024-11-20T05:14:18,468 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732079658468, completionTime=-1 2024-11-20T05:14:18,471 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T05:14:18,472 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T05:14:18,519 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-20T05:14:18,519 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732079718519 2024-11-20T05:14:18,519 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732079778519 2024-11-20T05:14:18,519 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 47 msec 2024-11-20T05:14:18,522 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,37927,1732079654666-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:18,525 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,37927,1732079654666-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:18,525 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,37927,1732079654666-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:18,527 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-51d5d2011a1c:37927, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:18,527 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:18,528 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:18,533 DEBUG [master/51d5d2011a1c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T05:14:18,557 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.002sec 2024-11-20T05:14:18,558 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T05:14:18,559 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T05:14:18,560 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T05:14:18,561 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T05:14:18,561 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T05:14:18,562 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,37927,1732079654666-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T05:14:18,562 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,37927,1732079654666-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T05:14:18,570 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T05:14:18,571 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T05:14:18,571 INFO [master/51d5d2011a1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51d5d2011a1c,37927,1732079654666-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T05:14:18,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:18,612 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T05:14:18,612 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T05:14:18,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:18,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:18,623 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d055ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:18,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:18,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:18,634 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:18,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:18,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:18,674 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=5464 2024-11-20T05:14:18,682 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T05:14:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1aaf840e 2024-11-20T05:14:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T05:14:18,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T05:14:18,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:14:18,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:14:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:18,709 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T05:14:18,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T05:14:18,711 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:18,713 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T05:14:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T05:14:18,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741835_1011 (size=1061) 2024-11-20T05:14:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T05:14:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T05:14:19,128 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:14:19,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741836_1012 (size=53) 2024-11-20T05:14:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T05:14:19,538 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:19,538 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 098be5f00fb47f2a54b5983aaea0749b, disabling compactions & flushes 2024-11-20T05:14:19,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. after waiting 0 ms 2024-11-20T05:14:19,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 098be5f00fb47f2a54b5983aaea0749b: Waiting for close lock at 1732079659538Disabling compacts and flushes for region at 1732079659538Disabling writes for close at 1732079659539 (+1 ms)Writing region close event to WAL at 1732079659539Closed at 1732079659539 2024-11-20T05:14:19,544 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T05:14:19,551 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732079659544"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732079659544"}]},"ts":"1732079659544"} 2024-11-20T05:14:19,557 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T05:14:19,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T05:14:19,562 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079659559"}]},"ts":"1732079659559"} 2024-11-20T05:14:19,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T05:14:19,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, ASSIGN}] 2024-11-20T05:14:19,571 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, ASSIGN 2024-11-20T05:14:19,573 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, ASSIGN; state=OFFLINE, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=false 2024-11-20T05:14:19,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=098be5f00fb47f2a54b5983aaea0749b, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:19,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, ASSIGN because future has completed 2024-11-20T05:14:19,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:14:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T05:14:19,902 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,902 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:14:19,902 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,903 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:19,903 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,903 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,905 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,909 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:19,910 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 098be5f00fb47f2a54b5983aaea0749b columnFamilyName A 2024-11-20T05:14:19,910 DEBUG [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:19,911 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.HStore(327): Store=098be5f00fb47f2a54b5983aaea0749b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:19,911 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,914 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:19,914 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 098be5f00fb47f2a54b5983aaea0749b columnFamilyName B 2024-11-20T05:14:19,915 DEBUG [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:19,916 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.HStore(327): Store=098be5f00fb47f2a54b5983aaea0749b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:19,916 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,918 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:19,919 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 098be5f00fb47f2a54b5983aaea0749b columnFamilyName C 2024-11-20T05:14:19,919 DEBUG [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:19,920 INFO [StoreOpener-098be5f00fb47f2a54b5983aaea0749b-1 {}] regionserver.HStore(327): Store=098be5f00fb47f2a54b5983aaea0749b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:19,921 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,921 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,922 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,923 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,925 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,925 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,926 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:14:19,928 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,932 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:14:19,933 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 098be5f00fb47f2a54b5983aaea0749b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61164176, jitterRate=-0.08858275413513184}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:14:19,933 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:19,934 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 098be5f00fb47f2a54b5983aaea0749b: Running coprocessor pre-open hook at 1732079659903Writing region info on filesystem at 1732079659903Initializing all the Stores at 1732079659905 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079659905Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079659905Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079659905Cleaning up temporary data from old regions at 1732079659925 (+20 ms)Running coprocessor post-open hooks at 1732079659933 (+8 ms)Region opened successfully at 1732079659934 (+1 ms) 2024-11-20T05:14:19,936 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., pid=6, masterSystemTime=1732079659889 2024-11-20T05:14:19,940 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,940 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:19,941 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=098be5f00fb47f2a54b5983aaea0749b, regionState=OPEN, openSeqNum=2, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:19,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:14:19,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T05:14:19,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 in 213 msec 2024-11-20T05:14:19,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T05:14:19,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, ASSIGN in 383 msec 2024-11-20T05:14:19,956 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T05:14:19,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079659956"}]},"ts":"1732079659956"} 2024-11-20T05:14:19,960 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T05:14:19,961 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T05:14:19,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2620 sec 2024-11-20T05:14:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T05:14:20,869 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:20,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f534466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297ba807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,892 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T05:14:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52a4d08 2024-11-20T05:14:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T05:14:20,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T05:14:20,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,904 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ae7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,908 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,912 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,916 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d79f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51273011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,933 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5476ceb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,935 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73797bde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,939 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657524e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,943 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c8a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,946 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b0da03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,948 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,952 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b54236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,955 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:20,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:20,958 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d424882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:20,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293d4ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:20,965 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,965 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,965 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,965 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,966 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,966 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,966 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,969 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,970 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:20,971 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:20,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:20,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T05:14:20,982 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T05:14:20,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:14:21,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:21,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:21,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:14:21,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:21,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:21,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:21,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:21,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:21,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T05:14:21,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:21,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/34649ebefd7940789554b5847cfa950e is 50, key is test_row_0/A:col10/1732079661047/Put/seqid=0 2024-11-20T05:14:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741837_1013 (size=12001) 2024-11-20T05:14:21,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/34649ebefd7940789554b5847cfa950e 2024-11-20T05:14:21,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079661166, max timestamp=1732079661218Num uniques 859; 2024-11-20T05:14:21,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079661166, max timestamp=1732079661222Num uniques 859; 2024-11-20T05:14:21,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:21,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:21,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079661166, max timestamp=1732079661218Num uniques 859; 2024-11-20T05:14:21,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T05:14:21,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079721303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079721308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079721313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079721317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079721321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/42120b23d4d34f63947705d51a49b13f is 50, key is test_row_0/B:col10/1732079661047/Put/seqid=0 2024-11-20T05:14:21,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741838_1014 (size=12001) 2024-11-20T05:14:21,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079721438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079721438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079721448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079721449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079721449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:21,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:21,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T05:14:21,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079721649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079721651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079721658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079721669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079721669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/42120b23d4d34f63947705d51a49b13f 2024-11-20T05:14:21,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/93c3ce83370a4024b83ebef6f0b15cdf is 50, key is test_row_0/C:col10/1732079661047/Put/seqid=0 2024-11-20T05:14:21,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:21,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741839_1015 (size=12001) 2024-11-20T05:14:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079721959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079721959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079721968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079721978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079721979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:21,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:22,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:22,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T05:14:22,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:22,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:22,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:22,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:22,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/93c3ce83370a4024b83ebef6f0b15cdf 2024-11-20T05:14:22,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/34649ebefd7940789554b5847cfa950e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/34649ebefd7940789554b5847cfa950e 2024-11-20T05:14:22,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/34649ebefd7940789554b5847cfa950e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T05:14:22,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/42120b23d4d34f63947705d51a49b13f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/42120b23d4d34f63947705d51a49b13f 2024-11-20T05:14:22,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:22,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:22,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/42120b23d4d34f63947705d51a49b13f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T05:14:22,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/93c3ce83370a4024b83ebef6f0b15cdf as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/93c3ce83370a4024b83ebef6f0b15cdf 2024-11-20T05:14:22,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/93c3ce83370a4024b83ebef6f0b15cdf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T05:14:22,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 098be5f00fb47f2a54b5983aaea0749b in 1299ms, sequenceid=13, compaction requested=false 2024-11-20T05:14:22,365 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T05:14:22,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:22,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T05:14:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:22,473 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-20T05:14:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:22,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:22,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:22,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:22,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:22,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:22,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3b673e29127d41ca8c5bb44025cd8563 is 50, key is test_row_0/A:col10/1732079661299/Put/seqid=0 2024-11-20T05:14:22,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741840_1016 (size=12001) 2024-11-20T05:14:22,512 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3b673e29127d41ca8c5bb44025cd8563 2024-11-20T05:14:22,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ada96c3bc5c248dfb141893065ef00e3 is 50, key is test_row_0/B:col10/1732079661299/Put/seqid=0 2024-11-20T05:14:22,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741841_1017 (size=12001) 2024-11-20T05:14:22,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079722567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079722571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079722576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079722582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079722583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079722699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079722699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079722751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079722751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079722752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079722947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079722958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,963 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ada96c3bc5c248dfb141893065ef00e3 2024-11-20T05:14:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079722980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079722981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079722981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:22,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:22,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/2ed50fc10b2644398a5cbc66e572fcd4 is 50, key is test_row_0/C:col10/1732079661299/Put/seqid=0 2024-11-20T05:14:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741842_1018 (size=12001) 2024-11-20T05:14:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T05:14:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079723257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079723268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079723299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079723305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079723305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,453 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/2ed50fc10b2644398a5cbc66e572fcd4 2024-11-20T05:14:23,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3b673e29127d41ca8c5bb44025cd8563 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3b673e29127d41ca8c5bb44025cd8563 2024-11-20T05:14:23,485 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3b673e29127d41ca8c5bb44025cd8563, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T05:14:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ada96c3bc5c248dfb141893065ef00e3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ada96c3bc5c248dfb141893065ef00e3 2024-11-20T05:14:23,501 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ada96c3bc5c248dfb141893065ef00e3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T05:14:23,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/2ed50fc10b2644398a5cbc66e572fcd4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2ed50fc10b2644398a5cbc66e572fcd4 2024-11-20T05:14:23,525 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2ed50fc10b2644398a5cbc66e572fcd4, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T05:14:23,528 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 098be5f00fb47f2a54b5983aaea0749b in 1056ms, sequenceid=49, compaction requested=false 2024-11-20T05:14:23,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:23,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:23,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T05:14:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T05:14:23,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T05:14:23,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5300 sec 2024-11-20T05:14:23,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5670 sec 2024-11-20T05:14:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:23,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T05:14:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:23,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/de5cb2c1657e45b39752ec77112891ae is 50, key is test_row_0/A:col10/1732079663779/Put/seqid=0 2024-11-20T05:14:23,830 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T05:14:23,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741843_1019 (size=14341) 2024-11-20T05:14:23,903 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T05:14:23,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079723898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079723899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079723900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079723902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079723910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079724018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079724018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079724018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079724018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079724038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079724228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079724228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079724228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/de5cb2c1657e45b39752ec77112891ae 2024-11-20T05:14:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079724229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079724247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/81c34a4357d44afcbd38e73cf03eb2b8 is 50, key is test_row_0/B:col10/1732079663779/Put/seqid=0 2024-11-20T05:14:24,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741844_1020 (size=12001) 2024-11-20T05:14:24,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079724537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079724548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079724548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079724548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079724558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:24,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/81c34a4357d44afcbd38e73cf03eb2b8 2024-11-20T05:14:24,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/bb53a1bc258c43fdbd1b9d1076577730 is 50, key is test_row_0/C:col10/1732079663779/Put/seqid=0 2024-11-20T05:14:24,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741845_1021 (size=12001) 2024-11-20T05:14:25,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T05:14:25,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T05:14:25,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T05:14:25,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T05:14:25,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T05:14:25,003 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T05:14:25,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T05:14:25,004 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T05:14:25,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079725058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:25,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079725058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079725067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079725067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:25,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079725068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:25,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/bb53a1bc258c43fdbd1b9d1076577730 2024-11-20T05:14:25,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/de5cb2c1657e45b39752ec77112891ae as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/de5cb2c1657e45b39752ec77112891ae 2024-11-20T05:14:25,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/de5cb2c1657e45b39752ec77112891ae, entries=200, sequenceid=62, filesize=14.0 K 2024-11-20T05:14:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/81c34a4357d44afcbd38e73cf03eb2b8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/81c34a4357d44afcbd38e73cf03eb2b8 2024-11-20T05:14:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T05:14:25,158 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:25,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T05:14:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T05:14:25,162 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:25,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/81c34a4357d44afcbd38e73cf03eb2b8, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T05:14:25,163 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:25,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:25,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/bb53a1bc258c43fdbd1b9d1076577730 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bb53a1bc258c43fdbd1b9d1076577730 2024-11-20T05:14:25,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bb53a1bc258c43fdbd1b9d1076577730, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T05:14:25,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 098be5f00fb47f2a54b5983aaea0749b in 1404ms, sequenceid=62, compaction requested=true 2024-11-20T05:14:25,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:25,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:25,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:25,191 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:25,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:25,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:25,191 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:25,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:25,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:25,199 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:25,201 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:25,201 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:25,201 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/42120b23d4d34f63947705d51a49b13f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ada96c3bc5c248dfb141893065ef00e3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/81c34a4357d44afcbd38e73cf03eb2b8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.2 K 2024-11-20T05:14:25,203 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 42120b23d4d34f63947705d51a49b13f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732079661047 2024-11-20T05:14:25,204 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ada96c3bc5c248dfb141893065ef00e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732079661299 2024-11-20T05:14:25,206 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 81c34a4357d44afcbd38e73cf03eb2b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079662560 2024-11-20T05:14:25,213 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:25,214 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:25,214 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:25,214 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/34649ebefd7940789554b5847cfa950e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3b673e29127d41ca8c5bb44025cd8563, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/de5cb2c1657e45b39752ec77112891ae] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=37.4 K 2024-11-20T05:14:25,217 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34649ebefd7940789554b5847cfa950e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732079661047 2024-11-20T05:14:25,218 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b673e29127d41ca8c5bb44025cd8563, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732079661299 2024-11-20T05:14:25,222 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5cb2c1657e45b39752ec77112891ae, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079662560 2024-11-20T05:14:25,259 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:25,261 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/7f329a70631a411bb3636ccb521a05c5 is 50, key is test_row_0/B:col10/1732079663779/Put/seqid=0 2024-11-20T05:14:25,265 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:25,266 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/a0bc1e3a313244359b98fd9b239f84bb is 50, key is test_row_0/A:col10/1732079663779/Put/seqid=0 2024-11-20T05:14:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T05:14:25,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741846_1022 (size=12104) 2024-11-20T05:14:25,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741847_1023 (size=12104) 2024-11-20T05:14:25,311 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/a0bc1e3a313244359b98fd9b239f84bb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a0bc1e3a313244359b98fd9b239f84bb 2024-11-20T05:14:25,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T05:14:25,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:25,319 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:25,337 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into a0bc1e3a313244359b98fd9b239f84bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:25,337 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/581050a32b134ca3877bbdfbddb25889 is 50, key is test_row_0/A:col10/1732079663839/Put/seqid=0 2024-11-20T05:14:25,340 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=13, startTime=1732079665187; duration=0sec 2024-11-20T05:14:25,340 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:25,341 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:25,341 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:25,344 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:25,344 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:25,344 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:25,345 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/93c3ce83370a4024b83ebef6f0b15cdf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2ed50fc10b2644398a5cbc66e572fcd4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bb53a1bc258c43fdbd1b9d1076577730] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.2 K 2024-11-20T05:14:25,346 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c3ce83370a4024b83ebef6f0b15cdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732079661047 2024-11-20T05:14:25,347 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ed50fc10b2644398a5cbc66e572fcd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732079661299 2024-11-20T05:14:25,348 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb53a1bc258c43fdbd1b9d1076577730, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079662560 2024-11-20T05:14:25,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741848_1024 (size=12001) 2024-11-20T05:14:25,375 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/581050a32b134ca3877bbdfbddb25889 2024-11-20T05:14:25,381 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:25,383 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/70fdd241071b472eb78d781fd03822a2 is 50, key is test_row_0/C:col10/1732079663779/Put/seqid=0 2024-11-20T05:14:25,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741849_1025 (size=12104) 2024-11-20T05:14:25,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/8fc0a8b4a464479a93d006f679f500df is 50, key is test_row_0/B:col10/1732079663839/Put/seqid=0 2024-11-20T05:14:25,430 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/70fdd241071b472eb78d781fd03822a2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/70fdd241071b472eb78d781fd03822a2 2024-11-20T05:14:25,444 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into 70fdd241071b472eb78d781fd03822a2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:25,444 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:25,444 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=13, startTime=1732079665191; duration=0sec 2024-11-20T05:14:25,444 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:25,444 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:25,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741850_1026 (size=12001) 2024-11-20T05:14:25,457 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/8fc0a8b4a464479a93d006f679f500df 2024-11-20T05:14:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T05:14:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/d86f996ed47742348ebec83a66f49149 is 50, key is test_row_0/C:col10/1732079663839/Put/seqid=0 2024-11-20T05:14:25,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741851_1027 (size=12001) 2024-11-20T05:14:25,550 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/d86f996ed47742348ebec83a66f49149 2024-11-20T05:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/581050a32b134ca3877bbdfbddb25889 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/581050a32b134ca3877bbdfbddb25889 2024-11-20T05:14:25,579 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/581050a32b134ca3877bbdfbddb25889, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T05:14:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/8fc0a8b4a464479a93d006f679f500df as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8fc0a8b4a464479a93d006f679f500df 2024-11-20T05:14:25,600 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8fc0a8b4a464479a93d006f679f500df, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T05:14:25,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/d86f996ed47742348ebec83a66f49149 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/d86f996ed47742348ebec83a66f49149 2024-11-20T05:14:25,628 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/d86f996ed47742348ebec83a66f49149, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T05:14:25,629 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 098be5f00fb47f2a54b5983aaea0749b in 310ms, sequenceid=82, compaction requested=false 2024-11-20T05:14:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T05:14:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T05:14:25,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T05:14:25,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 470 msec 2024-11-20T05:14:25,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 482 msec 2024-11-20T05:14:25,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/7f329a70631a411bb3636ccb521a05c5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/7f329a70631a411bb3636ccb521a05c5 2024-11-20T05:14:25,727 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into 7f329a70631a411bb3636ccb521a05c5(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:25,727 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:25,728 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=13, startTime=1732079665191; duration=0sec 2024-11-20T05:14:25,728 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:25,728 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T05:14:25,788 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:25,788 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T05:14:25,794 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T05:14:25,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:25,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T05:14:25,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T05:14:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:25,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:25,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:25,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T05:14:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T05:14:25,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T05:14:25,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-20T05:14:25,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 171 msec 2024-11-20T05:14:26,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:14:26,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:26,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:26,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:26,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:26,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:26,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:26,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/1f8c09d9938c4a5ea95ee54cd108a055 is 50, key is test_row_0/A:col10/1732079666081/Put/seqid=0 2024-11-20T05:14:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T05:14:26,108 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:26,109 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T05:14:26,113 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T05:14:26,116 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:26,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:26,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741852_1028 (size=12001) 2024-11-20T05:14:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732079666111, max timestamp=1732079666127Num uniques 859; 2024-11-20T05:14:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T05:14:26,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079666111, max timestamp=1732079666129Num uniques 859; 2024-11-20T05:14:26,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:26,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079666111, max timestamp=1732079666127Num uniques 859; 2024-11-20T05:14:26,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:26,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079726172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079726173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079726176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079726181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079726182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T05:14:26,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T05:14:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079726287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079726288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079726288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079726297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079726297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T05:14:26,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T05:14:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079726497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079726508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079726508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079726508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079726508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/1f8c09d9938c4a5ea95ee54cd108a055 2024-11-20T05:14:26,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/b03914510fbb4f0d9c31df47bb1e2ce8 is 50, key is test_row_0/B:col10/1732079666081/Put/seqid=0 2024-11-20T05:14:26,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741853_1029 (size=12001) 2024-11-20T05:14:26,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/b03914510fbb4f0d9c31df47bb1e2ce8 2024-11-20T05:14:26,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T05:14:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:26,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/f41dacb0cb394ec6a0401a9ff4b32339 is 50, key is test_row_0/C:col10/1732079666081/Put/seqid=0 2024-11-20T05:14:26,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741854_1030 (size=12001) 2024-11-20T05:14:26,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/f41dacb0cb394ec6a0401a9ff4b32339 2024-11-20T05:14:26,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/1f8c09d9938c4a5ea95ee54cd108a055 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1f8c09d9938c4a5ea95ee54cd108a055 2024-11-20T05:14:26,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1f8c09d9938c4a5ea95ee54cd108a055, entries=150, sequenceid=96, filesize=11.7 K 2024-11-20T05:14:26,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/b03914510fbb4f0d9c31df47bb1e2ce8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/b03914510fbb4f0d9c31df47bb1e2ce8 2024-11-20T05:14:26,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/b03914510fbb4f0d9c31df47bb1e2ce8, entries=150, sequenceid=96, filesize=11.7 K 2024-11-20T05:14:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/f41dacb0cb394ec6a0401a9ff4b32339 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/f41dacb0cb394ec6a0401a9ff4b32339 2024-11-20T05:14:26,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/f41dacb0cb394ec6a0401a9ff4b32339, entries=150, sequenceid=96, filesize=11.7 K 2024-11-20T05:14:26,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 098be5f00fb47f2a54b5983aaea0749b in 616ms, sequenceid=96, compaction requested=true 2024-11-20T05:14:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:26,704 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:26,704 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:26,707 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:26,707 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:26,707 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,707 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a0bc1e3a313244359b98fd9b239f84bb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/581050a32b134ca3877bbdfbddb25889, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1f8c09d9938c4a5ea95ee54cd108a055] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.3 K 2024-11-20T05:14:26,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:26,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:26,708 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,708 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/7f329a70631a411bb3636ccb521a05c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8fc0a8b4a464479a93d006f679f500df, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/b03914510fbb4f0d9c31df47bb1e2ce8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.3 K 2024-11-20T05:14:26,709 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0bc1e3a313244359b98fd9b239f84bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079662560 2024-11-20T05:14:26,713 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f329a70631a411bb3636ccb521a05c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079662560 2024-11-20T05:14:26,713 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 581050a32b134ca3877bbdfbddb25889, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732079663839 2024-11-20T05:14:26,714 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc0a8b4a464479a93d006f679f500df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732079663839 2024-11-20T05:14:26,714 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f8c09d9938c4a5ea95ee54cd108a055, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732079666077 2024-11-20T05:14:26,716 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b03914510fbb4f0d9c31df47bb1e2ce8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732079666077 2024-11-20T05:14:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T05:14:26,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T05:14:26,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,739 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T05:14:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:26,739 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:26,740 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/e2f8a9f77dfc4992954960ddfde37f21 is 50, key is test_row_0/A:col10/1732079666081/Put/seqid=0 2024-11-20T05:14:26,744 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:26,745 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/cde39dacf63646b6b537f81ae2613611 is 50, key is test_row_0/B:col10/1732079666081/Put/seqid=0 2024-11-20T05:14:26,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/1309cb7513e048ddb73a2493476fbbac is 50, key is test_row_0/A:col10/1732079666177/Put/seqid=0 2024-11-20T05:14:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741855_1031 (size=12207) 2024-11-20T05:14:26,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741856_1032 (size=12207) 2024-11-20T05:14:26,795 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/cde39dacf63646b6b537f81ae2613611 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cde39dacf63646b6b537f81ae2613611 2024-11-20T05:14:26,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741857_1033 (size=12151) 2024-11-20T05:14:26,809 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/1309cb7513e048ddb73a2493476fbbac 2024-11-20T05:14:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:26,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:26,814 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into cde39dacf63646b6b537f81ae2613611(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:26,814 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:26,814 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=13, startTime=1732079666704; duration=0sec 2024-11-20T05:14:26,815 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:26,815 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:26,815 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:26,830 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:26,830 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:26,830 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:26,831 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/70fdd241071b472eb78d781fd03822a2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/d86f996ed47742348ebec83a66f49149, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/f41dacb0cb394ec6a0401a9ff4b32339] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.3 K 2024-11-20T05:14:26,832 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 70fdd241071b472eb78d781fd03822a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079662560 2024-11-20T05:14:26,834 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d86f996ed47742348ebec83a66f49149, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732079663839 2024-11-20T05:14:26,835 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f41dacb0cb394ec6a0401a9ff4b32339, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732079666077 2024-11-20T05:14:26,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/8ac64d3df516410bb86f09cfa40f14ae is 50, key is test_row_0/B:col10/1732079666177/Put/seqid=0 2024-11-20T05:14:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079726833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079726834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079726838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079726839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079726841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741858_1034 (size=12151) 2024-11-20T05:14:26,874 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:26,875 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/cc5674b410824171b05aba1116042cae is 50, key is test_row_0/C:col10/1732079666081/Put/seqid=0 2024-11-20T05:14:26,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741859_1035 (size=12207) 2024-11-20T05:14:26,917 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/cc5674b410824171b05aba1116042cae as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc5674b410824171b05aba1116042cae 2024-11-20T05:14:26,931 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into cc5674b410824171b05aba1116042cae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:26,931 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:26,932 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=13, startTime=1732079666704; duration=0sec 2024-11-20T05:14:26,932 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:26,932 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:26,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079726957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079727167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079727167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079727167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079727178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079727178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,187 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/e2f8a9f77dfc4992954960ddfde37f21 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/e2f8a9f77dfc4992954960ddfde37f21 2024-11-20T05:14:27,196 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into e2f8a9f77dfc4992954960ddfde37f21(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:27,197 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:27,197 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=13, startTime=1732079666703; duration=0sec 2024-11-20T05:14:27,197 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:27,197 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T05:14:27,270 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/8ac64d3df516410bb86f09cfa40f14ae 2024-11-20T05:14:27,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/28d550fe05054c56a8b0244637546cfd is 50, key is test_row_0/C:col10/1732079666177/Put/seqid=0 2024-11-20T05:14:27,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741860_1036 (size=12151) 2024-11-20T05:14:27,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079727478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079727479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:27,703 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/28d550fe05054c56a8b0244637546cfd 2024-11-20T05:14:27,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/1309cb7513e048ddb73a2493476fbbac as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1309cb7513e048ddb73a2493476fbbac 2024-11-20T05:14:27,725 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1309cb7513e048ddb73a2493476fbbac, entries=150, sequenceid=132, filesize=11.9 K 2024-11-20T05:14:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/8ac64d3df516410bb86f09cfa40f14ae as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8ac64d3df516410bb86f09cfa40f14ae 2024-11-20T05:14:27,735 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8ac64d3df516410bb86f09cfa40f14ae, entries=150, sequenceid=132, filesize=11.9 K 2024-11-20T05:14:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/28d550fe05054c56a8b0244637546cfd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/28d550fe05054c56a8b0244637546cfd 2024-11-20T05:14:27,749 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/28d550fe05054c56a8b0244637546cfd, entries=150, sequenceid=132, filesize=11.9 K 2024-11-20T05:14:27,752 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 098be5f00fb47f2a54b5983aaea0749b in 1012ms, sequenceid=132, compaction requested=false 2024-11-20T05:14:27,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:27,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:27,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T05:14:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T05:14:27,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T05:14:27,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6380 sec 2024-11-20T05:14:27,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6540 sec 2024-11-20T05:14:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:28,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:14:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:28,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ed6a9b6aaa344c0f9983cefe7cef9ea9 is 50, key is test_row_0/A:col10/1732079667998/Put/seqid=0 2024-11-20T05:14:28,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741861_1037 (size=16931) 2024-11-20T05:14:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079728035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079728037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079728038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079728043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079728043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079728147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079728147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079728147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079728158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079728158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T05:14:28,257 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:28,257 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T05:14:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T05:14:28,261 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:28,262 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:28,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:28,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079728358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079728358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079728358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T05:14:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079728368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079728368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ed6a9b6aaa344c0f9983cefe7cef9ea9 2024-11-20T05:14:28,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/86dafeee404b4009b38ce331d4df7d57 is 50, key is test_row_0/B:col10/1732079667998/Put/seqid=0 2024-11-20T05:14:28,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741862_1038 (size=12151) 2024-11-20T05:14:28,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:28,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:28,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T05:14:28,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079728668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079728668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079728668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079728677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079728677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:28,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:28,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T05:14:28,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/86dafeee404b4009b38ce331d4df7d57 2024-11-20T05:14:28,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/057aff14a8a34b16940e791b86029ae1 is 50, key is test_row_0/C:col10/1732079667998/Put/seqid=0 2024-11-20T05:14:28,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741863_1039 (size=12151) 2024-11-20T05:14:29,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079729177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079729178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079729178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:29,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079729187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079729190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:29,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/057aff14a8a34b16940e791b86029ae1 2024-11-20T05:14:29,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ed6a9b6aaa344c0f9983cefe7cef9ea9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ed6a9b6aaa344c0f9983cefe7cef9ea9 2024-11-20T05:14:29,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:29,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ed6a9b6aaa344c0f9983cefe7cef9ea9, entries=250, sequenceid=148, filesize=16.5 K 2024-11-20T05:14:29,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/86dafeee404b4009b38ce331d4df7d57 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/86dafeee404b4009b38ce331d4df7d57 2024-11-20T05:14:29,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/86dafeee404b4009b38ce331d4df7d57, entries=150, sequenceid=148, filesize=11.9 K 2024-11-20T05:14:29,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/057aff14a8a34b16940e791b86029ae1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/057aff14a8a34b16940e791b86029ae1 2024-11-20T05:14:29,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/057aff14a8a34b16940e791b86029ae1, entries=150, sequenceid=148, filesize=11.9 K 2024-11-20T05:14:29,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 098be5f00fb47f2a54b5983aaea0749b in 1376ms, sequenceid=148, compaction requested=true 2024-11-20T05:14:29,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:29,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:29,377 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:29,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:29,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:29,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:29,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:29,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T05:14:29,377 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:29,378 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:29,378 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:29,379 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,379 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/e2f8a9f77dfc4992954960ddfde37f21, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1309cb7513e048ddb73a2493476fbbac, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ed6a9b6aaa344c0f9983cefe7cef9ea9] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=40.3 K 2024-11-20T05:14:29,379 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:29,379 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:29,379 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,380 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cde39dacf63646b6b537f81ae2613611, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8ac64d3df516410bb86f09cfa40f14ae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/86dafeee404b4009b38ce331d4df7d57] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.7 K 2024-11-20T05:14:29,380 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2f8a9f77dfc4992954960ddfde37f21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732079666077 2024-11-20T05:14:29,380 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting cde39dacf63646b6b537f81ae2613611, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732079666077 2024-11-20T05:14:29,381 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1309cb7513e048ddb73a2493476fbbac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079666171 2024-11-20T05:14:29,381 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ac64d3df516410bb86f09cfa40f14ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079666171 2024-11-20T05:14:29,381 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 86dafeee404b4009b38ce331d4df7d57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732079666838 2024-11-20T05:14:29,381 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed6a9b6aaa344c0f9983cefe7cef9ea9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732079666833 2024-11-20T05:14:29,396 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:29,397 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/b28f82c496344c94981bff2ab16ea820 is 50, key is test_row_0/A:col10/1732079667998/Put/seqid=0 2024-11-20T05:14:29,397 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T05:14:29,398 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/078fa6b79cf447a797aded503564ac0e is 50, key is test_row_0/B:col10/1732079667998/Put/seqid=0 2024-11-20T05:14:29,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741864_1040 (size=12459) 2024-11-20T05:14:29,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741865_1041 (size=12459) 2024-11-20T05:14:29,427 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/078fa6b79cf447a797aded503564ac0e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/078fa6b79cf447a797aded503564ac0e 2024-11-20T05:14:29,441 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into 078fa6b79cf447a797aded503564ac0e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:29,441 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:29,441 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=13, startTime=1732079669377; duration=0sec 2024-11-20T05:14:29,441 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:29,441 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:29,441 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:29,442 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:29,443 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:29,443 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,443 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc5674b410824171b05aba1116042cae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/28d550fe05054c56a8b0244637546cfd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/057aff14a8a34b16940e791b86029ae1] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.7 K 2024-11-20T05:14:29,443 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting cc5674b410824171b05aba1116042cae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732079666077 2024-11-20T05:14:29,444 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d550fe05054c56a8b0244637546cfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079666171 2024-11-20T05:14:29,449 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 057aff14a8a34b16940e791b86029ae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732079666838 2024-11-20T05:14:29,478 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#29 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:29,479 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/e466e05655dc43589e6d109780e35cca is 50, key is test_row_0/C:col10/1732079667998/Put/seqid=0 2024-11-20T05:14:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741866_1042 (size=12459) 2024-11-20T05:14:29,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T05:14:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:29,503 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:14:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:29,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/068843e28eaf447cb23647dffea08838 is 50, key is test_row_0/A:col10/1732079668037/Put/seqid=0 2024-11-20T05:14:29,523 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/e466e05655dc43589e6d109780e35cca as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/e466e05655dc43589e6d109780e35cca 2024-11-20T05:14:29,532 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into e466e05655dc43589e6d109780e35cca(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:29,532 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:29,532 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=13, startTime=1732079669377; duration=0sec 2024-11-20T05:14:29,533 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:29,533 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:29,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741867_1043 (size=12151) 2024-11-20T05:14:29,825 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/b28f82c496344c94981bff2ab16ea820 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b28f82c496344c94981bff2ab16ea820 2024-11-20T05:14:29,836 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into b28f82c496344c94981bff2ab16ea820(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:29,837 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:29,837 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=13, startTime=1732079669376; duration=0sec 2024-11-20T05:14:29,837 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:29,837 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:29,935 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/068843e28eaf447cb23647dffea08838 2024-11-20T05:14:29,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/6b6211b7a97748a7abf3639a27f90f85 is 50, key is test_row_0/B:col10/1732079668037/Put/seqid=0 2024-11-20T05:14:29,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741868_1044 (size=12151) 2024-11-20T05:14:29,964 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/6b6211b7a97748a7abf3639a27f90f85 2024-11-20T05:14:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/8655a56ba0fd456891d0c339ff5830d1 is 50, key is test_row_0/C:col10/1732079668037/Put/seqid=0 2024-11-20T05:14:29,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741869_1045 (size=12151) 2024-11-20T05:14:29,997 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/8655a56ba0fd456891d0c339ff5830d1 2024-11-20T05:14:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/068843e28eaf447cb23647dffea08838 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/068843e28eaf447cb23647dffea08838 2024-11-20T05:14:30,022 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/068843e28eaf447cb23647dffea08838, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T05:14:30,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/6b6211b7a97748a7abf3639a27f90f85 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/6b6211b7a97748a7abf3639a27f90f85 2024-11-20T05:14:30,031 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/6b6211b7a97748a7abf3639a27f90f85, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T05:14:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/8655a56ba0fd456891d0c339ff5830d1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8655a56ba0fd456891d0c339ff5830d1 2024-11-20T05:14:30,045 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8655a56ba0fd456891d0c339ff5830d1, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T05:14:30,046 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 098be5f00fb47f2a54b5983aaea0749b in 544ms, sequenceid=169, compaction requested=false 2024-11-20T05:14:30,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:30,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:30,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T05:14:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T05:14:30,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T05:14:30,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7870 sec 2024-11-20T05:14:30,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7950 sec 2024-11-20T05:14:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:30,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:14:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:30,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:30,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f632151228bc4d23954a40f6d2f43576 is 50, key is test_row_0/A:col10/1732079670197/Put/seqid=0 2024-11-20T05:14:30,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741870_1046 (size=12151) 2024-11-20T05:14:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079730229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079670219, max timestamp=1732079670234Num uniques 859; 2024-11-20T05:14:30,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:30,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732079670219, max timestamp=1732079670234Num uniques 859; 2024-11-20T05:14:30,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-20T05:14:30,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079670219, max timestamp=1732079670235Num uniques 859; 2024-11-20T05:14:30,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079730275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079730287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079730287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079730288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079730347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079730397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T05:14:30,407 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:30,408 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T05:14:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T05:14:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079730407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,411 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:30,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079730407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079730408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,412 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T05:14:30,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T05:14:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:30,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079730568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079730617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079730617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079730617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079730618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f632151228bc4d23954a40f6d2f43576 2024-11-20T05:14:30,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c1b31003a39c47a78de71aedecc97bf0 is 50, key is test_row_0/B:col10/1732079670197/Put/seqid=0 2024-11-20T05:14:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741871_1047 (size=12151) 2024-11-20T05:14:30,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T05:14:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:30,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T05:14:30,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T05:14:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:30,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:30,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079730877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079730928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079730928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079730928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079730929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:31,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T05:14:31,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:31,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:31,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:31,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:31,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T05:14:31,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c1b31003a39c47a78de71aedecc97bf0 2024-11-20T05:14:31,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/42b50fa72d3944938e80a72d789d6822 is 50, key is test_row_0/C:col10/1732079670197/Put/seqid=0 2024-11-20T05:14:31,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741872_1048 (size=12151) 2024-11-20T05:14:31,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/42b50fa72d3944938e80a72d789d6822 2024-11-20T05:14:31,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f632151228bc4d23954a40f6d2f43576 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f632151228bc4d23954a40f6d2f43576 2024-11-20T05:14:31,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f632151228bc4d23954a40f6d2f43576, entries=150, sequenceid=182, filesize=11.9 K 2024-11-20T05:14:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c1b31003a39c47a78de71aedecc97bf0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c1b31003a39c47a78de71aedecc97bf0 2024-11-20T05:14:31,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c1b31003a39c47a78de71aedecc97bf0, entries=150, sequenceid=182, filesize=11.9 K 2024-11-20T05:14:31,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/42b50fa72d3944938e80a72d789d6822 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/42b50fa72d3944938e80a72d789d6822 2024-11-20T05:14:31,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/42b50fa72d3944938e80a72d789d6822, entries=150, sequenceid=182, filesize=11.9 K 2024-11-20T05:14:31,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for 098be5f00fb47f2a54b5983aaea0749b in 909ms, sequenceid=182, compaction requested=true 2024-11-20T05:14:31,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:31,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:31,109 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:31,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:31,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:31,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:31,109 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:31,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:31,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:31,110 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:31,110 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:31,110 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:31,110 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:31,110 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:31,110 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:31,110 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b28f82c496344c94981bff2ab16ea820, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/068843e28eaf447cb23647dffea08838, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f632151228bc4d23954a40f6d2f43576] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.9 K 2024-11-20T05:14:31,110 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/078fa6b79cf447a797aded503564ac0e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/6b6211b7a97748a7abf3639a27f90f85, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c1b31003a39c47a78de71aedecc97bf0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.9 K 2024-11-20T05:14:31,111 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b28f82c496344c94981bff2ab16ea820, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732079666838 2024-11-20T05:14:31,111 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 078fa6b79cf447a797aded503564ac0e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732079666838 2024-11-20T05:14:31,111 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b6211b7a97748a7abf3639a27f90f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079668034 2024-11-20T05:14:31,111 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 068843e28eaf447cb23647dffea08838, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079668034 2024-11-20T05:14:31,112 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f632151228bc4d23954a40f6d2f43576, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732079670192 2024-11-20T05:14:31,112 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b31003a39c47a78de71aedecc97bf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732079670192 2024-11-20T05:14:31,123 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:31,123 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:31,124 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/cd48304eb2514877922283b11bfac809 is 50, key is test_row_0/B:col10/1732079670197/Put/seqid=0 2024-11-20T05:14:31,124 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/6f5134610117447cb55fc82b31828bad is 50, key is test_row_0/A:col10/1732079670197/Put/seqid=0 2024-11-20T05:14:31,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741874_1050 (size=12561) 2024-11-20T05:14:31,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741873_1049 (size=12561) 2024-11-20T05:14:31,156 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/6f5134610117447cb55fc82b31828bad as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/6f5134610117447cb55fc82b31828bad 2024-11-20T05:14:31,166 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into 6f5134610117447cb55fc82b31828bad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:31,166 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:31,167 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=13, startTime=1732079671109; duration=0sec 2024-11-20T05:14:31,167 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:31,167 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:31,167 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:31,168 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:31,169 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:31,169 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:31,170 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/e466e05655dc43589e6d109780e35cca, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8655a56ba0fd456891d0c339ff5830d1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/42b50fa72d3944938e80a72d789d6822] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=35.9 K 2024-11-20T05:14:31,171 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e466e05655dc43589e6d109780e35cca, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732079666838 2024-11-20T05:14:31,171 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8655a56ba0fd456891d0c339ff5830d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079668034 2024-11-20T05:14:31,172 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b50fa72d3944938e80a72d789d6822, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732079670192 2024-11-20T05:14:31,182 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:31,182 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/bceb7de47b464e479a411f6775acdace is 50, key is test_row_0/C:col10/1732079670197/Put/seqid=0 2024-11-20T05:14:31,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T05:14:31,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:31,185 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-20T05:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741875_1051 (size=12561) 2024-11-20T05:14:31,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/59786640e02c417b80ee3ccd2bba12fd is 50, key is test_row_0/A:col10/1732079670270/Put/seqid=0 2024-11-20T05:14:31,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741876_1052 (size=12151) 2024-11-20T05:14:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:31,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079731437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:31,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079731437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079731441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:31,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079731447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079731447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T05:14:31,552 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/cd48304eb2514877922283b11bfac809 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cd48304eb2514877922283b11bfac809 2024-11-20T05:14:31,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079731547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:31,561 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into cd48304eb2514877922283b11bfac809(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:31,561 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:31,561 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=13, startTime=1732079671109; duration=0sec 2024-11-20T05:14:31,561 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:31,561 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:31,598 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/59786640e02c417b80ee3ccd2bba12fd 2024-11-20T05:14:31,599 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/bceb7de47b464e479a411f6775acdace as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bceb7de47b464e479a411f6775acdace 2024-11-20T05:14:31,608 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into bceb7de47b464e479a411f6775acdace(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:31,608 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:31,608 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=13, startTime=1732079671109; duration=0sec 2024-11-20T05:14:31,608 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:31,608 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:31,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/a2949467534c48fea4f7cd614018b471 is 50, key is test_row_0/B:col10/1732079670270/Put/seqid=0 2024-11-20T05:14:31,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741877_1053 (size=12151) 2024-11-20T05:14:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079731757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,022 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/a2949467534c48fea4f7cd614018b471 2024-11-20T05:14:32,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/0402882bca6b4981abfbf8e1a4a2a25b is 50, key is test_row_0/C:col10/1732079670270/Put/seqid=0 2024-11-20T05:14:32,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741878_1054 (size=12151) 2024-11-20T05:14:32,039 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/0402882bca6b4981abfbf8e1a4a2a25b 2024-11-20T05:14:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/59786640e02c417b80ee3ccd2bba12fd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/59786640e02c417b80ee3ccd2bba12fd 2024-11-20T05:14:32,054 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/59786640e02c417b80ee3ccd2bba12fd, entries=150, sequenceid=220, filesize=11.9 K 2024-11-20T05:14:32,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/a2949467534c48fea4f7cd614018b471 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/a2949467534c48fea4f7cd614018b471 2024-11-20T05:14:32,062 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/a2949467534c48fea4f7cd614018b471, entries=150, sequenceid=220, filesize=11.9 K 2024-11-20T05:14:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/0402882bca6b4981abfbf8e1a4a2a25b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/0402882bca6b4981abfbf8e1a4a2a25b 2024-11-20T05:14:32,073 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/0402882bca6b4981abfbf8e1a4a2a25b, entries=150, sequenceid=220, filesize=11.9 K 2024-11-20T05:14:32,074 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for 098be5f00fb47f2a54b5983aaea0749b in 890ms, sequenceid=220, compaction requested=false 2024-11-20T05:14:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T05:14:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T05:14:32,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T05:14:32,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-20T05:14:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:32,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:14:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:32,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6720 sec 2024-11-20T05:14:32,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/29614e6e6d494b888274b084ee730e2f is 50, key is test_row_0/A:col10/1732079672077/Put/seqid=0 2024-11-20T05:14:32,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741879_1055 (size=12151) 2024-11-20T05:14:32,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/29614e6e6d494b888274b084ee730e2f 2024-11-20T05:14:32,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/1097b65da2f74076a6f7663926da2af8 is 50, key is test_row_0/B:col10/1732079672077/Put/seqid=0 2024-11-20T05:14:32,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741880_1056 (size=12151) 2024-11-20T05:14:32,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/1097b65da2f74076a6f7663926da2af8 2024-11-20T05:14:32,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/7bf75f7eef7d41ffa87a73a044b25ac8 is 50, key is test_row_0/C:col10/1732079672077/Put/seqid=0 2024-11-20T05:14:32,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741881_1057 (size=12151) 2024-11-20T05:14:32,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079672169, max timestamp=1732079672199Num uniques 859; 2024-11-20T05:14:32,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079672169, max timestamp=1732079672199Num uniques 859; 2024-11-20T05:14:32,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:14:32,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:14:32,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079672169, max timestamp=1732079672199Num uniques 859; 2024-11-20T05:14:32,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:14:32,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079732281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079732398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079732457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079732458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,463 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in 098be5f00fb47f2a54b5983aaea0749b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:32,463 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in 098be5f00fb47f2a54b5983aaea0749b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079732467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079732467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,473 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in 098be5f00fb47f2a54b5983aaea0749b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,473 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in 098be5f00fb47f2a54b5983aaea0749b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T05:14:32,557 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:32,558 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:32,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/7bf75f7eef7d41ffa87a73a044b25ac8 2024-11-20T05:14:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T05:14:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T05:14:32,561 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:32,563 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:32,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/29614e6e6d494b888274b084ee730e2f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/29614e6e6d494b888274b084ee730e2f 2024-11-20T05:14:32,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/29614e6e6d494b888274b084ee730e2f, entries=150, sequenceid=233, filesize=11.9 K 2024-11-20T05:14:32,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/1097b65da2f74076a6f7663926da2af8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/1097b65da2f74076a6f7663926da2af8 2024-11-20T05:14:32,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/1097b65da2f74076a6f7663926da2af8, entries=150, sequenceid=233, filesize=11.9 K 2024-11-20T05:14:32,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/7bf75f7eef7d41ffa87a73a044b25ac8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/7bf75f7eef7d41ffa87a73a044b25ac8 2024-11-20T05:14:32,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/7bf75f7eef7d41ffa87a73a044b25ac8, entries=150, sequenceid=233, filesize=11.9 K 2024-11-20T05:14:32,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 098be5f00fb47f2a54b5983aaea0749b in 518ms, sequenceid=233, compaction requested=true 2024-11-20T05:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:32,599 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:32,599 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:32,600 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:32,600 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:32,600 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:32,600 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,601 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cd48304eb2514877922283b11bfac809, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/a2949467534c48fea4f7cd614018b471, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/1097b65da2f74076a6f7663926da2af8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=36.0 K 2024-11-20T05:14:32,601 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:32,601 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,601 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/6f5134610117447cb55fc82b31828bad, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/59786640e02c417b80ee3ccd2bba12fd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/29614e6e6d494b888274b084ee730e2f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=36.0 K 2024-11-20T05:14:32,601 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting cd48304eb2514877922283b11bfac809, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732079670192 2024-11-20T05:14:32,601 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f5134610117447cb55fc82b31828bad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732079670192 2024-11-20T05:14:32,602 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a2949467534c48fea4f7cd614018b471, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732079670270 2024-11-20T05:14:32,602 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59786640e02c417b80ee3ccd2bba12fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732079670270 2024-11-20T05:14:32,602 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29614e6e6d494b888274b084ee730e2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732079671400 2024-11-20T05:14:32,603 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1097b65da2f74076a6f7663926da2af8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732079671400 2024-11-20T05:14:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:32,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T05:14:32,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:32,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:32,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:32,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:32,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:32,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:32,613 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:32,614 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ef004c4269bd426b9b0a20b2b593beb1 is 50, key is test_row_0/B:col10/1732079672077/Put/seqid=0 2024-11-20T05:14:32,614 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:32,614 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/4dd13685c59c4ae39bee7c4940f1b914 is 50, key is test_row_0/A:col10/1732079672077/Put/seqid=0 2024-11-20T05:14:32,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741882_1058 (size=12663) 2024-11-20T05:14:32,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741883_1059 (size=12663) 2024-11-20T05:14:32,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5 is 50, key is test_row_0/A:col10/1732079672608/Put/seqid=0 2024-11-20T05:14:32,630 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ef004c4269bd426b9b0a20b2b593beb1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ef004c4269bd426b9b0a20b2b593beb1 2024-11-20T05:14:32,639 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into ef004c4269bd426b9b0a20b2b593beb1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:32,639 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:32,639 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=13, startTime=1732079672599; duration=0sec 2024-11-20T05:14:32,639 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:32,639 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:32,639 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:32,640 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/4dd13685c59c4ae39bee7c4940f1b914 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/4dd13685c59c4ae39bee7c4940f1b914 2024-11-20T05:14:32,641 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:32,641 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:32,641 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,641 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bceb7de47b464e479a411f6775acdace, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/0402882bca6b4981abfbf8e1a4a2a25b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/7bf75f7eef7d41ffa87a73a044b25ac8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=36.0 K 2024-11-20T05:14:32,642 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bceb7de47b464e479a411f6775acdace, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732079670192 2024-11-20T05:14:32,642 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0402882bca6b4981abfbf8e1a4a2a25b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732079670270 2024-11-20T05:14:32,643 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf75f7eef7d41ffa87a73a044b25ac8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732079671400 2024-11-20T05:14:32,652 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into 4dd13685c59c4ae39bee7c4940f1b914(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:32,652 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:32,652 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=13, startTime=1732079672599; duration=0sec 2024-11-20T05:14:32,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741884_1060 (size=14741) 2024-11-20T05:14:32,660 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:32,660 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T05:14:32,670 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:32,670 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/2936461edd4649478ac5ea145cdbb6cb is 50, key is test_row_0/C:col10/1732079672077/Put/seqid=0 2024-11-20T05:14:32,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079732666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741885_1061 (size=12663) 2024-11-20T05:14:32,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:32,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079732777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:32,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:32,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T05:14:32,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079732988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:33,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5 2024-11-20T05:14:33,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/616cd7483d8c48ec96d12674ab7d52d3 is 50, key is test_row_0/B:col10/1732079672608/Put/seqid=0 2024-11-20T05:14:33,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741886_1062 (size=12301) 2024-11-20T05:14:33,091 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/2936461edd4649478ac5ea145cdbb6cb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2936461edd4649478ac5ea145cdbb6cb 2024-11-20T05:14:33,100 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into 2936461edd4649478ac5ea145cdbb6cb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:33,100 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:33,100 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=13, startTime=1732079672599; duration=0sec 2024-11-20T05:14:33,100 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:33,100 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:33,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:33,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:33,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T05:14:33,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079733298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:33,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:33,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:33,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/616cd7483d8c48ec96d12674ab7d52d3 2024-11-20T05:14:33,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:33,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:33,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/8670ba364af049df97736fa49278db71 is 50, key is test_row_0/C:col10/1732079672608/Put/seqid=0 2024-11-20T05:14:33,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741887_1063 (size=12301) 2024-11-20T05:14:33,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T05:14:33,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:33,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079733808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:33,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/8670ba364af049df97736fa49278db71 2024-11-20T05:14:33,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:33,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:33,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:33,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:33,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5 2024-11-20T05:14:33,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5, entries=200, sequenceid=269, filesize=14.4 K 2024-11-20T05:14:33,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/616cd7483d8c48ec96d12674ab7d52d3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/616cd7483d8c48ec96d12674ab7d52d3 2024-11-20T05:14:33,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/616cd7483d8c48ec96d12674ab7d52d3, entries=150, sequenceid=269, filesize=12.0 K 2024-11-20T05:14:33,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/8670ba364af049df97736fa49278db71 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8670ba364af049df97736fa49278db71 2024-11-20T05:14:34,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8670ba364af049df97736fa49278db71, entries=150, sequenceid=269, filesize=12.0 K 2024-11-20T05:14:34,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 098be5f00fb47f2a54b5983aaea0749b in 1405ms, sequenceid=269, compaction requested=false 2024-11-20T05:14:34,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:34,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T05:14:34,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:34,111 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:34,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/39cde753ea264fbdb1bbfb622e47e317 is 50, key is test_row_1/A:col10/1732079672644/Put/seqid=0 2024-11-20T05:14:34,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741888_1064 (size=9857) 2024-11-20T05:14:34,163 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/39cde753ea264fbdb1bbfb622e47e317 2024-11-20T05:14:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c9feaa508b1649e7932d653a04a0bb26 is 50, key is test_row_1/B:col10/1732079672644/Put/seqid=0 2024-11-20T05:14:34,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741889_1065 (size=9857) 2024-11-20T05:14:34,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079674543, max timestamp=1732079674558Num uniques 859; 2024-11-20T05:14:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:34,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079674543, max timestamp=1732079674558Num uniques 859; 2024-11-20T05:14:34,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:34,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079674543, max timestamp=1732079674558Num uniques 859; 2024-11-20T05:14:34,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:34,611 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c9feaa508b1649e7932d653a04a0bb26 2024-11-20T05:14:34,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079734606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079734614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079734614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079734616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/66035bbf8cea4c00870d1d2618717cd0 is 50, key is test_row_1/C:col10/1732079672644/Put/seqid=0 2024-11-20T05:14:34,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741890_1066 (size=9857) 2024-11-20T05:14:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T05:14:34,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079734727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079734727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079734727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079734727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079734827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,833 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in 098be5f00fb47f2a54b5983aaea0749b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079734937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079734937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079734938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079734939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,031 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/66035bbf8cea4c00870d1d2618717cd0 2024-11-20T05:14:35,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/39cde753ea264fbdb1bbfb622e47e317 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/39cde753ea264fbdb1bbfb622e47e317 2024-11-20T05:14:35,049 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/39cde753ea264fbdb1bbfb622e47e317, entries=100, sequenceid=282, filesize=9.6 K 2024-11-20T05:14:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c9feaa508b1649e7932d653a04a0bb26 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c9feaa508b1649e7932d653a04a0bb26 2024-11-20T05:14:35,057 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c9feaa508b1649e7932d653a04a0bb26, entries=100, sequenceid=282, filesize=9.6 K 2024-11-20T05:14:35,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/66035bbf8cea4c00870d1d2618717cd0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/66035bbf8cea4c00870d1d2618717cd0 2024-11-20T05:14:35,074 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/66035bbf8cea4c00870d1d2618717cd0, entries=100, sequenceid=282, filesize=9.6 K 2024-11-20T05:14:35,077 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 098be5f00fb47f2a54b5983aaea0749b in 966ms, sequenceid=282, compaction requested=true 2024-11-20T05:14:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T05:14:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T05:14:35,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T05:14:35,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5160 sec 2024-11-20T05:14:35,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.5280 sec 2024-11-20T05:14:35,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079674607, max timestamp=1732079675251Num uniques 1018; 2024-11-20T05:14:35,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:14:35,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079674607, max timestamp=1732079675251Num uniques 1018; 2024-11-20T05:14:35,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:14:35,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=71.56 KB heapSize=198.72 KB 2024-11-20T05:14:35,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:35,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:35,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:35,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:35,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:35,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T05:14:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:35,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f12dc2c931534965a511a4246f645b32 is 50, key is test_row_0/A:col10/1732079675254/Put/seqid=0 2024-11-20T05:14:35,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741891_1067 (size=17181) 2024-11-20T05:14:35,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f12dc2c931534965a511a4246f645b32 2024-11-20T05:14:35,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9b22ddf681dc44a3adaa162634192e71 is 50, key is test_row_0/B:col10/1732079675254/Put/seqid=0 2024-11-20T05:14:35,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741892_1068 (size=12301) 2024-11-20T05:14:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079735328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079735341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079735342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079735344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079735448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079735457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079735457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079735457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079735657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079735668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079735677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079735677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:35,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9b22ddf681dc44a3adaa162634192e71 2024-11-20T05:14:35,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/24499f614daa4d6297268d7589da4acb is 50, key is test_row_0/C:col10/1732079675254/Put/seqid=0 2024-11-20T05:14:35,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741893_1069 (size=12301) 2024-11-20T05:14:35,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/24499f614daa4d6297268d7589da4acb 2024-11-20T05:14:35,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f12dc2c931534965a511a4246f645b32 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f12dc2c931534965a511a4246f645b32 2024-11-20T05:14:35,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f12dc2c931534965a511a4246f645b32, entries=250, sequenceid=321, filesize=16.8 K 2024-11-20T05:14:35,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9b22ddf681dc44a3adaa162634192e71 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9b22ddf681dc44a3adaa162634192e71 2024-11-20T05:14:35,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9b22ddf681dc44a3adaa162634192e71, entries=150, sequenceid=321, filesize=12.0 K 2024-11-20T05:14:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/24499f614daa4d6297268d7589da4acb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/24499f614daa4d6297268d7589da4acb 2024-11-20T05:14:35,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/24499f614daa4d6297268d7589da4acb, entries=150, sequenceid=321, filesize=12.0 K 2024-11-20T05:14:35,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for 098be5f00fb47f2a54b5983aaea0749b in 544ms, sequenceid=321, compaction requested=true 2024-11-20T05:14:35,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:35,803 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:14:35,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:35,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:35,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:35,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:35,803 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:14:35,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:35,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:35,804 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:14:35,804 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:35,805 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:35,805 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/4dd13685c59c4ae39bee7c4940f1b914, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/39cde753ea264fbdb1bbfb622e47e317, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f12dc2c931534965a511a4246f645b32] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=53.2 K 2024-11-20T05:14:35,806 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47122 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:14:35,806 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:35,807 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:35,807 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ef004c4269bd426b9b0a20b2b593beb1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/616cd7483d8c48ec96d12674ab7d52d3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c9feaa508b1649e7932d653a04a0bb26, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9b22ddf681dc44a3adaa162634192e71] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=46.0 K 2024-11-20T05:14:35,807 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dd13685c59c4ae39bee7c4940f1b914, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732079671400 2024-11-20T05:14:35,807 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ef004c4269bd426b9b0a20b2b593beb1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732079671400 2024-11-20T05:14:35,807 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c50c9cd2bc4b2ba9d7cf853d6c88c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732079672265 2024-11-20T05:14:35,808 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 616cd7483d8c48ec96d12674ab7d52d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732079672265 2024-11-20T05:14:35,808 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39cde753ea264fbdb1bbfb622e47e317, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732079672644 2024-11-20T05:14:35,808 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c9feaa508b1649e7932d653a04a0bb26, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732079672644 2024-11-20T05:14:35,808 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12dc2c931534965a511a4246f645b32, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732079674607 2024-11-20T05:14:35,809 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b22ddf681dc44a3adaa162634192e71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732079674607 2024-11-20T05:14:35,831 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:35,831 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f20a4d958a184c0dbb98e49b2c0fe738 is 50, key is test_row_0/A:col10/1732079675254/Put/seqid=0 2024-11-20T05:14:35,835 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:35,835 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/485f5d9451744606b9479a115d13f357 is 50, key is test_row_0/B:col10/1732079675254/Put/seqid=0 2024-11-20T05:14:35,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741894_1070 (size=12949) 2024-11-20T05:14:35,850 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/f20a4d958a184c0dbb98e49b2c0fe738 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f20a4d958a184c0dbb98e49b2c0fe738 2024-11-20T05:14:35,860 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into f20a4d958a184c0dbb98e49b2c0fe738(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:35,860 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:35,860 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=12, startTime=1732079675803; duration=0sec 2024-11-20T05:14:35,860 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:35,860 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:35,861 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:14:35,862 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47122 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:14:35,863 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:35,863 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:35,863 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2936461edd4649478ac5ea145cdbb6cb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8670ba364af049df97736fa49278db71, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/66035bbf8cea4c00870d1d2618717cd0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/24499f614daa4d6297268d7589da4acb] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=46.0 K 2024-11-20T05:14:35,864 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2936461edd4649478ac5ea145cdbb6cb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732079671400 2024-11-20T05:14:35,865 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8670ba364af049df97736fa49278db71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732079672265 2024-11-20T05:14:35,866 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66035bbf8cea4c00870d1d2618717cd0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732079672644 2024-11-20T05:14:35,866 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24499f614daa4d6297268d7589da4acb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732079674607 2024-11-20T05:14:35,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741895_1071 (size=12949) 2024-11-20T05:14:35,890 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:35,891 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/cc88be0e49da46e7939b63a1b2d53341 is 50, key is test_row_0/C:col10/1732079675254/Put/seqid=0 2024-11-20T05:14:35,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741896_1072 (size=12949) 2024-11-20T05:14:35,935 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/cc88be0e49da46e7939b63a1b2d53341 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc88be0e49da46e7939b63a1b2d53341 2024-11-20T05:14:35,956 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into cc88be0e49da46e7939b63a1b2d53341(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:35,956 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:35,956 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=12, startTime=1732079675803; duration=0sec 2024-11-20T05:14:35,956 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:35,956 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:35,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:14:35,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079675290, max timestamp=1732079675981Num uniques 859; 2024-11-20T05:14:35,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079675290, max timestamp=1732079675981Num uniques 859; 2024-11-20T05:14:35,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079675290, max timestamp=1732079675981Num uniques 859; 2024-11-20T05:14:35,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:14:35,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:14:35,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:14:35,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:35,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:35,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:35,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:35,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:35,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:36,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3f971edb9d7a4289bd7d59fdd90ca0d6 is 50, key is test_row_0/A:col10/1732079675326/Put/seqid=0 2024-11-20T05:14:36,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741897_1073 (size=14741) 2024-11-20T05:14:36,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3f971edb9d7a4289bd7d59fdd90ca0d6 2024-11-20T05:14:36,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732079676022, max timestamp=1732079676041Num uniques 859; 2024-11-20T05:14:36,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T05:14:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079676026, max timestamp=1732079676041Num uniques 859; 2024-11-20T05:14:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:14:36,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079676026, max timestamp=1732079676041Num uniques 859; 2024-11-20T05:14:36,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:14:36,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/526d9dcbf4ee4c379469a522ee4df66a is 50, key is test_row_0/B:col10/1732079675326/Put/seqid=0 2024-11-20T05:14:36,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741898_1074 (size=12301) 2024-11-20T05:14:36,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/526d9dcbf4ee4c379469a522ee4df66a 2024-11-20T05:14:36,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732079676073, max timestamp=1732079676102Num uniques 1168; 2024-11-20T05:14:36,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-20T05:14:36,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732079676073, max timestamp=1732079676102Num uniques 1018; 2024-11-20T05:14:36,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-20T05:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732079676073, max timestamp=1732079676102Num uniques 1068; 2024-11-20T05:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-20T05:14:36,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/264bb33862f94cd18fc708375752ca33 is 50, key is test_row_0/C:col10/1732079675326/Put/seqid=0 2024-11-20T05:14:36,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741899_1075 (size=12301) 2024-11-20T05:14:36,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/264bb33862f94cd18fc708375752ca33 2024-11-20T05:14:36,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3f971edb9d7a4289bd7d59fdd90ca0d6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3f971edb9d7a4289bd7d59fdd90ca0d6 2024-11-20T05:14:36,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3f971edb9d7a4289bd7d59fdd90ca0d6, entries=200, sequenceid=344, filesize=14.4 K 2024-11-20T05:14:36,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/526d9dcbf4ee4c379469a522ee4df66a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/526d9dcbf4ee4c379469a522ee4df66a 2024-11-20T05:14:36,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/526d9dcbf4ee4c379469a522ee4df66a, entries=150, sequenceid=344, filesize=12.0 K 2024-11-20T05:14:36,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/264bb33862f94cd18fc708375752ca33 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/264bb33862f94cd18fc708375752ca33 2024-11-20T05:14:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732079676132, max timestamp=1732079676157Num uniques 1177; 2024-11-20T05:14:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-11-20T05:14:36,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732079676132, max timestamp=1732079676157Num uniques 1177; 2024-11-20T05:14:36,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-11-20T05:14:36,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732079676132, max timestamp=1732079676157Num uniques 1177; 2024-11-20T05:14:36,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-11-20T05:14:36,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/264bb33862f94cd18fc708375752ca33, entries=150, sequenceid=344, filesize=12.0 K 2024-11-20T05:14:36,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=80.51 KB/82440 for 098be5f00fb47f2a54b5983aaea0749b in 192ms, sequenceid=344, compaction requested=false 2024-11-20T05:14:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:36,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=100.63 KB heapSize=269.63 KB 2024-11-20T05:14:36,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:36,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:36,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:36,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:36,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:36,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:36,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/d8794bc2530e488b8c244d3f0ebf0d6b is 50, key is test_row_0/A:col10/1732079676185/Put/seqid=0 2024-11-20T05:14:36,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079736210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079736212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079736214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079736216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741900_1076 (size=17181) 2024-11-20T05:14:36,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/d8794bc2530e488b8c244d3f0ebf0d6b 2024-11-20T05:14:36,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9ec155c74f0647e0889f8c135119f22e is 50, key is test_row_0/B:col10/1732079676185/Put/seqid=0 2024-11-20T05:14:36,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741901_1077 (size=12301) 2024-11-20T05:14:36,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9ec155c74f0647e0889f8c135119f22e 2024-11-20T05:14:36,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/db26121f7c26463e9ea069e48fad2c7f is 50, key is test_row_0/C:col10/1732079676185/Put/seqid=0 2024-11-20T05:14:36,294 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/485f5d9451744606b9479a115d13f357 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/485f5d9451744606b9479a115d13f357 2024-11-20T05:14:36,306 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into 485f5d9451744606b9479a115d13f357(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:36,306 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:36,306 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=12, startTime=1732079675803; duration=0sec 2024-11-20T05:14:36,306 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:36,306 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:36,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079736328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079736328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079736328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079736328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741902_1078 (size=12301) 2024-11-20T05:14:36,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/db26121f7c26463e9ea069e48fad2c7f 2024-11-20T05:14:36,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/d8794bc2530e488b8c244d3f0ebf0d6b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/d8794bc2530e488b8c244d3f0ebf0d6b 2024-11-20T05:14:36,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/d8794bc2530e488b8c244d3f0ebf0d6b, entries=250, sequenceid=408, filesize=16.8 K 2024-11-20T05:14:36,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9ec155c74f0647e0889f8c135119f22e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9ec155c74f0647e0889f8c135119f22e 2024-11-20T05:14:36,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9ec155c74f0647e0889f8c135119f22e, entries=150, sequenceid=408, filesize=12.0 K 2024-11-20T05:14:36,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/db26121f7c26463e9ea069e48fad2c7f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/db26121f7c26463e9ea069e48fad2c7f 2024-11-20T05:14:36,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/db26121f7c26463e9ea069e48fad2c7f, entries=150, sequenceid=408, filesize=12.0 K 2024-11-20T05:14:36,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~289.50 KB/296448, currentSize=80.51 KB/82440 for 098be5f00fb47f2a54b5983aaea0749b in 215ms, sequenceid=408, compaction requested=true 2024-11-20T05:14:36,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:36,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:36,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:36,402 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:36,402 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:36,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:36,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:36,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:36,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:36,403 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:36,403 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:36,403 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:36,404 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f20a4d958a184c0dbb98e49b2c0fe738, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3f971edb9d7a4289bd7d59fdd90ca0d6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/d8794bc2530e488b8c244d3f0ebf0d6b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=43.8 K 2024-11-20T05:14:36,404 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:36,404 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f20a4d958a184c0dbb98e49b2c0fe738, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732079674607 2024-11-20T05:14:36,404 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:36,404 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:36,404 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/485f5d9451744606b9479a115d13f357, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/526d9dcbf4ee4c379469a522ee4df66a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9ec155c74f0647e0889f8c135119f22e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=36.7 K 2024-11-20T05:14:36,404 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f971edb9d7a4289bd7d59fdd90ca0d6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732079675290 2024-11-20T05:14:36,405 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 485f5d9451744606b9479a115d13f357, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732079674607 2024-11-20T05:14:36,405 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8794bc2530e488b8c244d3f0ebf0d6b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732079676167 2024-11-20T05:14:36,406 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 526d9dcbf4ee4c379469a522ee4df66a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732079675326 2024-11-20T05:14:36,406 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec155c74f0647e0889f8c135119f22e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732079676173 2024-11-20T05:14:36,423 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:36,423 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ab3b4f07607a40eab5e8a8bb95e67c7e is 50, key is test_row_0/B:col10/1732079676185/Put/seqid=0 2024-11-20T05:14:36,428 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:36,429 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/fe475ee5b6ae4d8d959dbb139bb89154 is 50, key is test_row_0/A:col10/1732079676185/Put/seqid=0 2024-11-20T05:14:36,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741903_1079 (size=13051) 2024-11-20T05:14:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741904_1080 (size=13051) 2024-11-20T05:14:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:36,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:14:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:36,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/590f5b155c2a4050a79a65f6c0d2bb8c is 50, key is test_row_0/A:col10/1732079676542/Put/seqid=0 2024-11-20T05:14:36,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741905_1081 (size=12301) 2024-11-20T05:14:36,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/590f5b155c2a4050a79a65f6c0d2bb8c 2024-11-20T05:14:36,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079736571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/58db140f491d416ab2128aa3eb65ed6a is 50, key is test_row_0/B:col10/1732079676542/Put/seqid=0 2024-11-20T05:14:36,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079736573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079736573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079736574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741906_1082 (size=12301) 2024-11-20T05:14:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079736687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079736687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079736697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079736697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T05:14:36,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:36,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T05:14:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T05:14:36,720 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:36,721 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:36,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T05:14:36,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079736847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,848 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in 098be5f00fb47f2a54b5983aaea0749b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:36,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T05:14:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:36,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:36,891 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ab3b4f07607a40eab5e8a8bb95e67c7e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ab3b4f07607a40eab5e8a8bb95e67c7e 2024-11-20T05:14:36,900 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into ab3b4f07607a40eab5e8a8bb95e67c7e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:36,900 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:36,901 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=13, startTime=1732079676402; duration=0sec 2024-11-20T05:14:36,902 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:36,902 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:36,903 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:36,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079736898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079736898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,911 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:36,911 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:36,911 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:36,911 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc88be0e49da46e7939b63a1b2d53341, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/264bb33862f94cd18fc708375752ca33, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/db26121f7c26463e9ea069e48fad2c7f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=36.7 K 2024-11-20T05:14:36,913 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting cc88be0e49da46e7939b63a1b2d53341, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732079674607 2024-11-20T05:14:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,914 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 264bb33862f94cd18fc708375752ca33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732079675326 2024-11-20T05:14:36,914 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/fe475ee5b6ae4d8d959dbb139bb89154 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/fe475ee5b6ae4d8d959dbb139bb89154 2024-11-20T05:14:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079736908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079736908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:36,916 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting db26121f7c26463e9ea069e48fad2c7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732079676173 2024-11-20T05:14:36,925 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into fe475ee5b6ae4d8d959dbb139bb89154(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:36,925 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:36,925 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=13, startTime=1732079676401; duration=0sec 2024-11-20T05:14:36,925 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:36,925 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:36,933 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:36,934 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/979d7de089184172969dc1aa50c890fc is 50, key is test_row_0/C:col10/1732079676185/Put/seqid=0 2024-11-20T05:14:36,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741907_1083 (size=13051) 2024-11-20T05:14:36,954 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/979d7de089184172969dc1aa50c890fc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/979d7de089184172969dc1aa50c890fc 2024-11-20T05:14:36,963 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into 979d7de089184172969dc1aa50c890fc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:36,963 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:36,963 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=13, startTime=1732079676402; duration=0sec 2024-11-20T05:14:36,963 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:36,963 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:36,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/58db140f491d416ab2128aa3eb65ed6a 2024-11-20T05:14:37,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/dc98e54e282a41df8c64fd278cc60af6 is 50, key is test_row_0/C:col10/1732079676542/Put/seqid=0 2024-11-20T05:14:37,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T05:14:37,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:37,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:37,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:37,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:37,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T05:14:37,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741908_1084 (size=12301) 2024-11-20T05:14:37,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/dc98e54e282a41df8c64fd278cc60af6 2024-11-20T05:14:37,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/590f5b155c2a4050a79a65f6c0d2bb8c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/590f5b155c2a4050a79a65f6c0d2bb8c 2024-11-20T05:14:37,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/590f5b155c2a4050a79a65f6c0d2bb8c, entries=150, sequenceid=425, filesize=12.0 K 2024-11-20T05:14:37,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/58db140f491d416ab2128aa3eb65ed6a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/58db140f491d416ab2128aa3eb65ed6a 2024-11-20T05:14:37,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/58db140f491d416ab2128aa3eb65ed6a, entries=150, sequenceid=425, filesize=12.0 K 2024-11-20T05:14:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/dc98e54e282a41df8c64fd278cc60af6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/dc98e54e282a41df8c64fd278cc60af6 2024-11-20T05:14:37,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/dc98e54e282a41df8c64fd278cc60af6, entries=150, sequenceid=425, filesize=12.0 K 2024-11-20T05:14:37,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 098be5f00fb47f2a54b5983aaea0749b in 541ms, sequenceid=425, compaction requested=false 2024-11-20T05:14:37,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:37,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T05:14:37,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:37,186 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:14:37,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:37,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:37,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:37,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ab776260b3a0411396d8e65b21f07f13 is 50, key is test_row_0/A:col10/1732079676571/Put/seqid=0 2024-11-20T05:14:37,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741909_1085 (size=9857) 2024-11-20T05:14:37,208 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ab776260b3a0411396d8e65b21f07f13 2024-11-20T05:14:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:37,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/aca4870e856043bca99b2144d53efabf is 50, key is test_row_0/B:col10/1732079676571/Put/seqid=0 2024-11-20T05:14:37,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741910_1086 (size=9857) 2024-11-20T05:14:37,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079737238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079737238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079737241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079737245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T05:14:37,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079737358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079737360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079737360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079737360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079737567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079737567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079737568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079737568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,640 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/aca4870e856043bca99b2144d53efabf 2024-11-20T05:14:37,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/be9c12e9b93744a4a98e6d6f6c869b4c is 50, key is test_row_0/C:col10/1732079676571/Put/seqid=0 2024-11-20T05:14:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741911_1087 (size=9857) 2024-11-20T05:14:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T05:14:37,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079737877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079737877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079737878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:37,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079737878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,056 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/be9c12e9b93744a4a98e6d6f6c869b4c 2024-11-20T05:14:38,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ab776260b3a0411396d8e65b21f07f13 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ab776260b3a0411396d8e65b21f07f13 2024-11-20T05:14:38,067 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ab776260b3a0411396d8e65b21f07f13, entries=100, sequenceid=444, filesize=9.6 K 2024-11-20T05:14:38,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/aca4870e856043bca99b2144d53efabf as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/aca4870e856043bca99b2144d53efabf 2024-11-20T05:14:38,074 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/aca4870e856043bca99b2144d53efabf, entries=100, sequenceid=444, filesize=9.6 K 2024-11-20T05:14:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/be9c12e9b93744a4a98e6d6f6c869b4c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/be9c12e9b93744a4a98e6d6f6c869b4c 2024-11-20T05:14:38,081 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/be9c12e9b93744a4a98e6d6f6c869b4c, entries=100, sequenceid=444, filesize=9.6 K 2024-11-20T05:14:38,082 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 098be5f00fb47f2a54b5983aaea0749b in 895ms, sequenceid=444, compaction requested=true 2024-11-20T05:14:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T05:14:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T05:14:38,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T05:14:38,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3620 sec 2024-11-20T05:14:38,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3680 sec 2024-11-20T05:14:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:38,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:14:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:38,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/91d60e0f649b449da22ae3aad83036da is 50, key is test_row_0/A:col10/1732079677239/Put/seqid=0 2024-11-20T05:14:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079738417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741912_1088 (size=14741) 2024-11-20T05:14:38,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/91d60e0f649b449da22ae3aad83036da 2024-11-20T05:14:38,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079738424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079738426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079738426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ebeb2c1907bf41fb83967be2f4a6f193 is 50, key is test_row_0/B:col10/1732079677239/Put/seqid=0 2024-11-20T05:14:38,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741913_1089 (size=12301) 2024-11-20T05:14:38,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079738538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079738538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079738547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079738547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079738747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079738747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079738758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079738758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T05:14:38,868 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:38,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-20T05:14:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T05:14:38,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:38,871 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:38,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:38,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ebeb2c1907bf41fb83967be2f4a6f193 2024-11-20T05:14:38,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/c76f0e625a364fef9f8d666423199176 is 50, key is test_row_0/C:col10/1732079677239/Put/seqid=0 2024-11-20T05:14:38,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741914_1090 (size=12301) 2024-11-20T05:14:38,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/c76f0e625a364fef9f8d666423199176 2024-11-20T05:14:38,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/91d60e0f649b449da22ae3aad83036da as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/91d60e0f649b449da22ae3aad83036da 2024-11-20T05:14:38,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/91d60e0f649b449da22ae3aad83036da, entries=200, sequenceid=461, filesize=14.4 K 2024-11-20T05:14:38,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ebeb2c1907bf41fb83967be2f4a6f193 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ebeb2c1907bf41fb83967be2f4a6f193 2024-11-20T05:14:38,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ebeb2c1907bf41fb83967be2f4a6f193, entries=150, sequenceid=461, filesize=12.0 K 2024-11-20T05:14:38,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/c76f0e625a364fef9f8d666423199176 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/c76f0e625a364fef9f8d666423199176 2024-11-20T05:14:38,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/c76f0e625a364fef9f8d666423199176, entries=150, sequenceid=461, filesize=12.0 K 2024-11-20T05:14:38,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 098be5f00fb47f2a54b5983aaea0749b in 543ms, sequenceid=461, compaction requested=true 2024-11-20T05:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:38,933 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:38,933 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:38,934 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49950 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:14:38,934 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:38,935 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:14:38,935 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:38,935 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:38,935 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:38,935 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/fe475ee5b6ae4d8d959dbb139bb89154, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/590f5b155c2a4050a79a65f6c0d2bb8c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ab776260b3a0411396d8e65b21f07f13, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/91d60e0f649b449da22ae3aad83036da] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=48.8 K 2024-11-20T05:14:38,935 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ab3b4f07607a40eab5e8a8bb95e67c7e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/58db140f491d416ab2128aa3eb65ed6a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/aca4870e856043bca99b2144d53efabf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ebeb2c1907bf41fb83967be2f4a6f193] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=46.4 K 2024-11-20T05:14:38,935 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe475ee5b6ae4d8d959dbb139bb89154, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732079676173 2024-11-20T05:14:38,935 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ab3b4f07607a40eab5e8a8bb95e67c7e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732079676173 2024-11-20T05:14:38,936 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590f5b155c2a4050a79a65f6c0d2bb8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732079676201 2024-11-20T05:14:38,936 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 58db140f491d416ab2128aa3eb65ed6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732079676201 2024-11-20T05:14:38,936 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting aca4870e856043bca99b2144d53efabf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732079676571 2024-11-20T05:14:38,936 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab776260b3a0411396d8e65b21f07f13, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732079676571 2024-11-20T05:14:38,937 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ebeb2c1907bf41fb83967be2f4a6f193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732079677236 2024-11-20T05:14:38,937 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d60e0f649b449da22ae3aad83036da, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732079677236 2024-11-20T05:14:38,961 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:38,963 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/0d63754c050440b79ef2619e42eedbf4 is 50, key is test_row_0/A:col10/1732079677239/Put/seqid=0 2024-11-20T05:14:38,975 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:38,976 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/f2ddcc4cd4384a8ea33cd87c32c93d55 is 50, key is test_row_0/B:col10/1732079677239/Put/seqid=0 2024-11-20T05:14:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T05:14:38,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741915_1091 (size=13187) 2024-11-20T05:14:38,991 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/0d63754c050440b79ef2619e42eedbf4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/0d63754c050440b79ef2619e42eedbf4 2024-11-20T05:14:38,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741916_1092 (size=13187) 2024-11-20T05:14:38,998 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into 0d63754c050440b79ef2619e42eedbf4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:38,998 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:38,998 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=12, startTime=1732079678933; duration=0sec 2024-11-20T05:14:38,998 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:38,998 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:38,998 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:14:39,000 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:14:39,000 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:39,000 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:39,000 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/979d7de089184172969dc1aa50c890fc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/dc98e54e282a41df8c64fd278cc60af6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/be9c12e9b93744a4a98e6d6f6c869b4c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/c76f0e625a364fef9f8d666423199176] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=46.4 K 2024-11-20T05:14:39,001 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979d7de089184172969dc1aa50c890fc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732079676173 2024-11-20T05:14:39,001 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc98e54e282a41df8c64fd278cc60af6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732079676201 2024-11-20T05:14:39,002 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting be9c12e9b93744a4a98e6d6f6c869b4c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732079676571 2024-11-20T05:14:39,002 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c76f0e625a364fef9f8d666423199176, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732079677236 2024-11-20T05:14:39,018 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:39,018 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/86099ab8983643cfae4e8c437b259890 is 50, key is test_row_0/C:col10/1732079677239/Put/seqid=0 2024-11-20T05:14:39,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T05:14:39,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:39,024 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:14:39,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/b43067a370bd49feb08abd6f3a30774b is 50, key is test_row_0/A:col10/1732079678422/Put/seqid=0 2024-11-20T05:14:39,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741917_1093 (size=13187) 2024-11-20T05:14:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:39,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. as already flushing 2024-11-20T05:14:39,072 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/86099ab8983643cfae4e8c437b259890 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/86099ab8983643cfae4e8c437b259890 2024-11-20T05:14:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741918_1094 (size=12301) 2024-11-20T05:14:39,081 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into 86099ab8983643cfae4e8c437b259890(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:39,081 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:39,081 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=12, startTime=1732079678933; duration=0sec 2024-11-20T05:14:39,081 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:39,081 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:39,084 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/b43067a370bd49feb08abd6f3a30774b 2024-11-20T05:14:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9a7018352ce94dfd9535ff7d9075d628 is 50, key is test_row_0/B:col10/1732079678422/Put/seqid=0 2024-11-20T05:14:39,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079739112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079739116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079739117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079739117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741919_1095 (size=12301) 2024-11-20T05:14:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T05:14:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079739228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079739228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079739228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079739238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,403 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/f2ddcc4cd4384a8ea33cd87c32c93d55 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/f2ddcc4cd4384a8ea33cd87c32c93d55 2024-11-20T05:14:39,416 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into f2ddcc4cd4384a8ea33cd87c32c93d55(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:39,416 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:39,416 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=12, startTime=1732079678933; duration=0sec 2024-11-20T05:14:39,416 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:39,417 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:39,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079739437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079739447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079739448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079739448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T05:14:39,564 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9a7018352ce94dfd9535ff7d9075d628 2024-11-20T05:14:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/756f128489074b64821d43ddd4aced08 is 50, key is test_row_0/C:col10/1732079678422/Put/seqid=0 2024-11-20T05:14:39,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741920_1096 (size=12301) 2024-11-20T05:14:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079739747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079739767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079739767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079739768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:39,983 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/756f128489074b64821d43ddd4aced08 2024-11-20T05:14:39,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/b43067a370bd49feb08abd6f3a30774b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b43067a370bd49feb08abd6f3a30774b 2024-11-20T05:14:39,996 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b43067a370bd49feb08abd6f3a30774b, entries=150, sequenceid=478, filesize=12.0 K 2024-11-20T05:14:39,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/9a7018352ce94dfd9535ff7d9075d628 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9a7018352ce94dfd9535ff7d9075d628 2024-11-20T05:14:40,005 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9a7018352ce94dfd9535ff7d9075d628, entries=150, sequenceid=478, filesize=12.0 K 2024-11-20T05:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/756f128489074b64821d43ddd4aced08 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/756f128489074b64821d43ddd4aced08 2024-11-20T05:14:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T05:14:40,012 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/756f128489074b64821d43ddd4aced08, entries=150, sequenceid=478, filesize=12.0 K 2024-11-20T05:14:40,013 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 098be5f00fb47f2a54b5983aaea0749b in 989ms, sequenceid=478, compaction requested=false 2024-11-20T05:14:40,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:40,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:40,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-20T05:14:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-20T05:14:40,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T05:14:40,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1440 sec 2024-11-20T05:14:40,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1500 sec 2024-11-20T05:14:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:40,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:14:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:40,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/a6109d02b6494bf4aef4ad3958d97a31 is 50, key is test_row_0/A:col10/1732079679112/Put/seqid=0 2024-11-20T05:14:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741921_1097 (size=14741) 2024-11-20T05:14:40,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079740291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079740292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079740294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079740294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079740397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079740407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079740407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079740417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079740608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079740617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079740617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079740627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/a6109d02b6494bf4aef4ad3958d97a31 2024-11-20T05:14:40,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c094d85761574ddfb625944cf713e28c is 50, key is test_row_0/B:col10/1732079679112/Put/seqid=0 2024-11-20T05:14:40,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741922_1098 (size=12301) 2024-11-20T05:14:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37716 deadline: 1732079740887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,889 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in 098be5f00fb47f2a54b5983aaea0749b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079740917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079740927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079740927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079740937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:40,971 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T05:14:40,971 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:40,971 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T05:14:40,971 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:40,972 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T05:14:40,972 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:40,973 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T05:14:40,973 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:40,975 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,975 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,975 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,975 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,976 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,976 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,976 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:40,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:40,976 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:40,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:40,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T05:14:41,028 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:41,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c094d85761574ddfb625944cf713e28c 2024-11-20T05:14:41,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/708b2d9809e047ed8669b41da38dce99 is 50, key is test_row_0/C:col10/1732079679112/Put/seqid=0 2024-11-20T05:14:41,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741923_1099 (size=12301) 2024-11-20T05:14:41,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37722 deadline: 1732079741427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:41,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:41,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37708 deadline: 1732079741438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37746 deadline: 1732079741438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1732079741447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:14:41,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/708b2d9809e047ed8669b41da38dce99 2024-11-20T05:14:41,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/a6109d02b6494bf4aef4ad3958d97a31 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a6109d02b6494bf4aef4ad3958d97a31 2024-11-20T05:14:41,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a6109d02b6494bf4aef4ad3958d97a31, entries=200, sequenceid=497, filesize=14.4 K 2024-11-20T05:14:41,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/c094d85761574ddfb625944cf713e28c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c094d85761574ddfb625944cf713e28c 2024-11-20T05:14:41,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c094d85761574ddfb625944cf713e28c, entries=150, sequenceid=497, filesize=12.0 K 2024-11-20T05:14:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/708b2d9809e047ed8669b41da38dce99 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/708b2d9809e047ed8669b41da38dce99 2024-11-20T05:14:41,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/708b2d9809e047ed8669b41da38dce99, entries=150, sequenceid=497, filesize=12.0 K 2024-11-20T05:14:41,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 098be5f00fb47f2a54b5983aaea0749b in 1281ms, sequenceid=497, compaction requested=true 2024-11-20T05:14:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:14:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:41,540 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:14:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:41,540 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 098be5f00fb47f2a54b5983aaea0749b:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:14:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:41,541 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:41,541 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:41,541 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/A is initiating minor compaction (all files) 2024-11-20T05:14:41,541 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/B is initiating minor compaction (all files) 2024-11-20T05:14:41,541 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/A in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:41,541 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/B in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:41,541 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/0d63754c050440b79ef2619e42eedbf4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b43067a370bd49feb08abd6f3a30774b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a6109d02b6494bf4aef4ad3958d97a31] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=39.3 K 2024-11-20T05:14:41,541 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/f2ddcc4cd4384a8ea33cd87c32c93d55, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9a7018352ce94dfd9535ff7d9075d628, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c094d85761574ddfb625944cf713e28c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=36.9 K 2024-11-20T05:14:41,542 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d63754c050440b79ef2619e42eedbf4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732079677236 2024-11-20T05:14:41,542 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ddcc4cd4384a8ea33cd87c32c93d55, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732079677236 2024-11-20T05:14:41,542 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43067a370bd49feb08abd6f3a30774b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732079678406 2024-11-20T05:14:41,542 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a7018352ce94dfd9535ff7d9075d628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732079678406 2024-11-20T05:14:41,542 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6109d02b6494bf4aef4ad3958d97a31, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732079679112 2024-11-20T05:14:41,542 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c094d85761574ddfb625944cf713e28c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732079679112 2024-11-20T05:14:41,551 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:41,552 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:41,552 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ac0387241495453bbca3acf4919704c2 is 50, key is test_row_0/A:col10/1732079679112/Put/seqid=0 2024-11-20T05:14:41,552 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ac70431954f245e0a50d9f31b57e6056 is 50, key is test_row_0/B:col10/1732079679112/Put/seqid=0 2024-11-20T05:14:41,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741925_1101 (size=13289) 2024-11-20T05:14:41,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741924_1100 (size=13289) 2024-11-20T05:14:41,968 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/ac0387241495453bbca3acf4919704c2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ac0387241495453bbca3acf4919704c2 2024-11-20T05:14:41,968 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ac70431954f245e0a50d9f31b57e6056 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ac70431954f245e0a50d9f31b57e6056 2024-11-20T05:14:41,974 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/B of 098be5f00fb47f2a54b5983aaea0749b into ac70431954f245e0a50d9f31b57e6056(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:41,974 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/A of 098be5f00fb47f2a54b5983aaea0749b into ac0387241495453bbca3acf4919704c2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:41,975 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:41,975 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:41,975 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/B, priority=13, startTime=1732079681540; duration=0sec 2024-11-20T05:14:41,975 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/A, priority=13, startTime=1732079681540; duration=0sec 2024-11-20T05:14:41,975 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:41,975 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:B 2024-11-20T05:14:41,975 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:14:41,975 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:A 2024-11-20T05:14:41,975 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:14:41,976 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:14:41,976 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 098be5f00fb47f2a54b5983aaea0749b/C is initiating minor compaction (all files) 2024-11-20T05:14:41,976 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 098be5f00fb47f2a54b5983aaea0749b/C in TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:41,977 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/86099ab8983643cfae4e8c437b259890, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/756f128489074b64821d43ddd4aced08, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/708b2d9809e047ed8669b41da38dce99] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp, totalSize=36.9 K 2024-11-20T05:14:41,977 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 86099ab8983643cfae4e8c437b259890, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732079677236 2024-11-20T05:14:41,977 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 756f128489074b64821d43ddd4aced08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732079678406 2024-11-20T05:14:41,978 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 708b2d9809e047ed8669b41da38dce99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732079679112 2024-11-20T05:14:41,985 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 098be5f00fb47f2a54b5983aaea0749b#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:14:41,986 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/5e56c60ffbd24ff69e0d643fa570a72a is 50, key is test_row_0/C:col10/1732079679112/Put/seqid=0 2024-11-20T05:14:41,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741926_1102 (size=13289) 2024-11-20T05:14:42,400 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/5e56c60ffbd24ff69e0d643fa570a72a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/5e56c60ffbd24ff69e0d643fa570a72a 2024-11-20T05:14:42,409 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 098be5f00fb47f2a54b5983aaea0749b/C of 098be5f00fb47f2a54b5983aaea0749b into 5e56c60ffbd24ff69e0d643fa570a72a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:14:42,409 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:42,409 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., storeName=098be5f00fb47f2a54b5983aaea0749b/C, priority=13, startTime=1732079681540; duration=0sec 2024-11-20T05:14:42,409 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:14:42,409 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 098be5f00fb47f2a54b5983aaea0749b:C 2024-11-20T05:14:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:42,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:14:42,440 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T05:14:42,440 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:42,440 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:42,440 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:42,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:42,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:42,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:42,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:42,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:42,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/eda3b86b8dbd4bc5b7dff6d178e8bea9 is 50, key is test_row_0/A:col10/1732079680293/Put/seqid=0 2024-11-20T05:14:42,450 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T05:14:42,450 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T05:14:42,450 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:42,450 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,450 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:42,450 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,450 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:42,450 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741927_1103 (size=12301) 2024-11-20T05:14:42,459 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T05:14:42,459 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:42,459 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,459 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:42,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:42,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/eda3b86b8dbd4bc5b7dff6d178e8bea9 2024-11-20T05:14:42,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ed0039a1f6d3401184462ef17a75baf5 is 50, key is test_row_0/B:col10/1732079680293/Put/seqid=0 2024-11-20T05:14:42,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741928_1104 (size=12301) 2024-11-20T05:14:43,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ed0039a1f6d3401184462ef17a75baf5 2024-11-20T05:14:43,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/49bc8e522ba3444bbbe77a70b01dcba0 is 50, key is test_row_0/C:col10/1732079680293/Put/seqid=0 2024-11-20T05:14:43,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741929_1105 (size=12301) 2024-11-20T05:14:43,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/49bc8e522ba3444bbbe77a70b01dcba0 2024-11-20T05:14:43,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/eda3b86b8dbd4bc5b7dff6d178e8bea9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/eda3b86b8dbd4bc5b7dff6d178e8bea9 2024-11-20T05:14:43,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/eda3b86b8dbd4bc5b7dff6d178e8bea9, entries=150, sequenceid=516, filesize=12.0 K 2024-11-20T05:14:43,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/ed0039a1f6d3401184462ef17a75baf5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ed0039a1f6d3401184462ef17a75baf5 2024-11-20T05:14:43,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ed0039a1f6d3401184462ef17a75baf5, entries=150, sequenceid=516, filesize=12.0 K 2024-11-20T05:14:43,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/49bc8e522ba3444bbbe77a70b01dcba0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/49bc8e522ba3444bbbe77a70b01dcba0 2024-11-20T05:14:43,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/49bc8e522ba3444bbbe77a70b01dcba0, entries=150, sequenceid=516, filesize=12.0 K 2024-11-20T05:14:43,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for 098be5f00fb47f2a54b5983aaea0749b in 1286ms, sequenceid=516, compaction requested=false 2024-11-20T05:14:43,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 098be5f00fb47f2a54b5983aaea0749b: 2024-11-20T05:14:43,788 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T05:14:50,932 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T05:14:50,932 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:14:50,932 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:50,933 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:50,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:50,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T05:14:50,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-20T05:14:50,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2659 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2631 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-20T05:14:50,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3536 rows 2024-11-20T05:14:50,935 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1181 2024-11-20T05:14:50,935 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3542 rows 2024-11-20T05:14:50,935 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:14:50,935 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:14:50,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:50,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:50,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:14:50,942 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T05:14:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T05:14:50,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079690949"}]},"ts":"1732079690949"} 2024-11-20T05:14:50,953 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T05:14:50,953 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T05:14:50,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:14:50,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, UNASSIGN}] 2024-11-20T05:14:50,962 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, UNASSIGN 2024-11-20T05:14:50,963 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=098be5f00fb47f2a54b5983aaea0749b, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:50,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, UNASSIGN because future has completed 2024-11-20T05:14:50,967 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:14:50,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:14:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T05:14:51,127 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:51,127 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:14:51,128 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 098be5f00fb47f2a54b5983aaea0749b, disabling compactions & flushes 2024-11-20T05:14:51,128 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:51,128 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:51,129 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. after waiting 0 ms 2024-11-20T05:14:51,129 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:51,129 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 098be5f00fb47f2a54b5983aaea0749b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T05:14:51,130 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=A 2024-11-20T05:14:51,130 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:51,130 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=B 2024-11-20T05:14:51,130 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:51,130 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 098be5f00fb47f2a54b5983aaea0749b, store=C 2024-11-20T05:14:51,130 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:51,141 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3d10d9de13814fcab5f8e4055f7e64c7 is 50, key is test_row_0/A:col10/1732079682448/Put/seqid=0 2024-11-20T05:14:51,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741930_1106 (size=12301) 2024-11-20T05:14:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T05:14:51,547 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3d10d9de13814fcab5f8e4055f7e64c7 2024-11-20T05:14:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T05:14:51,588 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/4bc83d38bb614d288d7a413ebf59c573 is 50, key is test_row_0/B:col10/1732079682448/Put/seqid=0 2024-11-20T05:14:51,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741931_1107 (size=12301) 2024-11-20T05:14:51,994 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/4bc83d38bb614d288d7a413ebf59c573 2024-11-20T05:14:52,005 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/10cb2cacaadd431bb0d0ed532afdb55b is 50, key is test_row_0/C:col10/1732079682448/Put/seqid=0 2024-11-20T05:14:52,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741932_1108 (size=12301) 2024-11-20T05:14:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T05:14:52,410 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/10cb2cacaadd431bb0d0ed532afdb55b 2024-11-20T05:14:52,423 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/A/3d10d9de13814fcab5f8e4055f7e64c7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3d10d9de13814fcab5f8e4055f7e64c7 2024-11-20T05:14:52,430 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3d10d9de13814fcab5f8e4055f7e64c7, entries=150, sequenceid=523, filesize=12.0 K 2024-11-20T05:14:52,431 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/B/4bc83d38bb614d288d7a413ebf59c573 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/4bc83d38bb614d288d7a413ebf59c573 2024-11-20T05:14:52,436 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/4bc83d38bb614d288d7a413ebf59c573, entries=150, sequenceid=523, filesize=12.0 K 2024-11-20T05:14:52,437 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/.tmp/C/10cb2cacaadd431bb0d0ed532afdb55b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/10cb2cacaadd431bb0d0ed532afdb55b 2024-11-20T05:14:52,442 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/10cb2cacaadd431bb0d0ed532afdb55b, entries=150, sequenceid=523, filesize=12.0 K 2024-11-20T05:14:52,443 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 098be5f00fb47f2a54b5983aaea0749b in 1314ms, sequenceid=523, compaction requested=true 2024-11-20T05:14:52,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/34649ebefd7940789554b5847cfa950e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3b673e29127d41ca8c5bb44025cd8563, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/de5cb2c1657e45b39752ec77112891ae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a0bc1e3a313244359b98fd9b239f84bb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/581050a32b134ca3877bbdfbddb25889, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/e2f8a9f77dfc4992954960ddfde37f21, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1f8c09d9938c4a5ea95ee54cd108a055, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1309cb7513e048ddb73a2493476fbbac, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ed6a9b6aaa344c0f9983cefe7cef9ea9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b28f82c496344c94981bff2ab16ea820, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/068843e28eaf447cb23647dffea08838, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/6f5134610117447cb55fc82b31828bad, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f632151228bc4d23954a40f6d2f43576, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/59786640e02c417b80ee3ccd2bba12fd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/4dd13685c59c4ae39bee7c4940f1b914, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/29614e6e6d494b888274b084ee730e2f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/39cde753ea264fbdb1bbfb622e47e317, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f12dc2c931534965a511a4246f645b32, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f20a4d958a184c0dbb98e49b2c0fe738, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3f971edb9d7a4289bd7d59fdd90ca0d6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/d8794bc2530e488b8c244d3f0ebf0d6b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/fe475ee5b6ae4d8d959dbb139bb89154, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/590f5b155c2a4050a79a65f6c0d2bb8c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ab776260b3a0411396d8e65b21f07f13, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/91d60e0f649b449da22ae3aad83036da, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/0d63754c050440b79ef2619e42eedbf4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b43067a370bd49feb08abd6f3a30774b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a6109d02b6494bf4aef4ad3958d97a31] to archive 2024-11-20T05:14:52,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:14:52,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/34649ebefd7940789554b5847cfa950e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/34649ebefd7940789554b5847cfa950e 2024-11-20T05:14:52,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3b673e29127d41ca8c5bb44025cd8563 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3b673e29127d41ca8c5bb44025cd8563 2024-11-20T05:14:52,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/de5cb2c1657e45b39752ec77112891ae to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/de5cb2c1657e45b39752ec77112891ae 2024-11-20T05:14:52,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a0bc1e3a313244359b98fd9b239f84bb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a0bc1e3a313244359b98fd9b239f84bb 2024-11-20T05:14:52,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/581050a32b134ca3877bbdfbddb25889 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/581050a32b134ca3877bbdfbddb25889 2024-11-20T05:14:52,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/e2f8a9f77dfc4992954960ddfde37f21 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/e2f8a9f77dfc4992954960ddfde37f21 2024-11-20T05:14:52,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1f8c09d9938c4a5ea95ee54cd108a055 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1f8c09d9938c4a5ea95ee54cd108a055 2024-11-20T05:14:52,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1309cb7513e048ddb73a2493476fbbac to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/1309cb7513e048ddb73a2493476fbbac 2024-11-20T05:14:52,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ed6a9b6aaa344c0f9983cefe7cef9ea9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ed6a9b6aaa344c0f9983cefe7cef9ea9 2024-11-20T05:14:52,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b28f82c496344c94981bff2ab16ea820 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b28f82c496344c94981bff2ab16ea820 2024-11-20T05:14:52,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/068843e28eaf447cb23647dffea08838 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/068843e28eaf447cb23647dffea08838 2024-11-20T05:14:52,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/6f5134610117447cb55fc82b31828bad to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/6f5134610117447cb55fc82b31828bad 2024-11-20T05:14:52,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f632151228bc4d23954a40f6d2f43576 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f632151228bc4d23954a40f6d2f43576 2024-11-20T05:14:52,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/59786640e02c417b80ee3ccd2bba12fd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/59786640e02c417b80ee3ccd2bba12fd 2024-11-20T05:14:52,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/4dd13685c59c4ae39bee7c4940f1b914 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/4dd13685c59c4ae39bee7c4940f1b914 2024-11-20T05:14:52,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/29614e6e6d494b888274b084ee730e2f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/29614e6e6d494b888274b084ee730e2f 2024-11-20T05:14:52,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/69c50c9cd2bc4b2ba9d7cf853d6c88c5 2024-11-20T05:14:52,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/39cde753ea264fbdb1bbfb622e47e317 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/39cde753ea264fbdb1bbfb622e47e317 2024-11-20T05:14:52,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f12dc2c931534965a511a4246f645b32 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f12dc2c931534965a511a4246f645b32 2024-11-20T05:14:52,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f20a4d958a184c0dbb98e49b2c0fe738 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/f20a4d958a184c0dbb98e49b2c0fe738 2024-11-20T05:14:52,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3f971edb9d7a4289bd7d59fdd90ca0d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3f971edb9d7a4289bd7d59fdd90ca0d6 2024-11-20T05:14:52,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/d8794bc2530e488b8c244d3f0ebf0d6b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/d8794bc2530e488b8c244d3f0ebf0d6b 2024-11-20T05:14:52,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/fe475ee5b6ae4d8d959dbb139bb89154 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/fe475ee5b6ae4d8d959dbb139bb89154 2024-11-20T05:14:52,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/590f5b155c2a4050a79a65f6c0d2bb8c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/590f5b155c2a4050a79a65f6c0d2bb8c 2024-11-20T05:14:52,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ab776260b3a0411396d8e65b21f07f13 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ab776260b3a0411396d8e65b21f07f13 2024-11-20T05:14:52,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/91d60e0f649b449da22ae3aad83036da to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/91d60e0f649b449da22ae3aad83036da 2024-11-20T05:14:52,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/0d63754c050440b79ef2619e42eedbf4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/0d63754c050440b79ef2619e42eedbf4 2024-11-20T05:14:52,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b43067a370bd49feb08abd6f3a30774b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/b43067a370bd49feb08abd6f3a30774b 2024-11-20T05:14:52,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a6109d02b6494bf4aef4ad3958d97a31 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/a6109d02b6494bf4aef4ad3958d97a31 2024-11-20T05:14:52,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/42120b23d4d34f63947705d51a49b13f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ada96c3bc5c248dfb141893065ef00e3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/7f329a70631a411bb3636ccb521a05c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/81c34a4357d44afcbd38e73cf03eb2b8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8fc0a8b4a464479a93d006f679f500df, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cde39dacf63646b6b537f81ae2613611, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/b03914510fbb4f0d9c31df47bb1e2ce8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8ac64d3df516410bb86f09cfa40f14ae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/078fa6b79cf447a797aded503564ac0e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/86dafeee404b4009b38ce331d4df7d57, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/6b6211b7a97748a7abf3639a27f90f85, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cd48304eb2514877922283b11bfac809, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c1b31003a39c47a78de71aedecc97bf0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/a2949467534c48fea4f7cd614018b471, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ef004c4269bd426b9b0a20b2b593beb1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/1097b65da2f74076a6f7663926da2af8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/616cd7483d8c48ec96d12674ab7d52d3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c9feaa508b1649e7932d653a04a0bb26, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/485f5d9451744606b9479a115d13f357, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9b22ddf681dc44a3adaa162634192e71, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/526d9dcbf4ee4c379469a522ee4df66a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ab3b4f07607a40eab5e8a8bb95e67c7e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9ec155c74f0647e0889f8c135119f22e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/58db140f491d416ab2128aa3eb65ed6a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/aca4870e856043bca99b2144d53efabf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/f2ddcc4cd4384a8ea33cd87c32c93d55, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ebeb2c1907bf41fb83967be2f4a6f193, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9a7018352ce94dfd9535ff7d9075d628, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c094d85761574ddfb625944cf713e28c] to archive 2024-11-20T05:14:52,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:14:52,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/42120b23d4d34f63947705d51a49b13f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/42120b23d4d34f63947705d51a49b13f 2024-11-20T05:14:52,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ada96c3bc5c248dfb141893065ef00e3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ada96c3bc5c248dfb141893065ef00e3 2024-11-20T05:14:52,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/7f329a70631a411bb3636ccb521a05c5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/7f329a70631a411bb3636ccb521a05c5 2024-11-20T05:14:52,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/81c34a4357d44afcbd38e73cf03eb2b8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/81c34a4357d44afcbd38e73cf03eb2b8 2024-11-20T05:14:52,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8fc0a8b4a464479a93d006f679f500df to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8fc0a8b4a464479a93d006f679f500df 2024-11-20T05:14:52,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cde39dacf63646b6b537f81ae2613611 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cde39dacf63646b6b537f81ae2613611 2024-11-20T05:14:52,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/b03914510fbb4f0d9c31df47bb1e2ce8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/b03914510fbb4f0d9c31df47bb1e2ce8 2024-11-20T05:14:52,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8ac64d3df516410bb86f09cfa40f14ae to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/8ac64d3df516410bb86f09cfa40f14ae 2024-11-20T05:14:52,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/078fa6b79cf447a797aded503564ac0e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/078fa6b79cf447a797aded503564ac0e 2024-11-20T05:14:52,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/86dafeee404b4009b38ce331d4df7d57 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/86dafeee404b4009b38ce331d4df7d57 2024-11-20T05:14:52,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/6b6211b7a97748a7abf3639a27f90f85 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/6b6211b7a97748a7abf3639a27f90f85 2024-11-20T05:14:52,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cd48304eb2514877922283b11bfac809 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/cd48304eb2514877922283b11bfac809 2024-11-20T05:14:52,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c1b31003a39c47a78de71aedecc97bf0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c1b31003a39c47a78de71aedecc97bf0 2024-11-20T05:14:52,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/a2949467534c48fea4f7cd614018b471 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/a2949467534c48fea4f7cd614018b471 2024-11-20T05:14:52,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ef004c4269bd426b9b0a20b2b593beb1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ef004c4269bd426b9b0a20b2b593beb1 2024-11-20T05:14:52,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/1097b65da2f74076a6f7663926da2af8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/1097b65da2f74076a6f7663926da2af8 2024-11-20T05:14:52,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/616cd7483d8c48ec96d12674ab7d52d3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/616cd7483d8c48ec96d12674ab7d52d3 2024-11-20T05:14:52,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c9feaa508b1649e7932d653a04a0bb26 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c9feaa508b1649e7932d653a04a0bb26 2024-11-20T05:14:52,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/485f5d9451744606b9479a115d13f357 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/485f5d9451744606b9479a115d13f357 2024-11-20T05:14:52,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9b22ddf681dc44a3adaa162634192e71 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9b22ddf681dc44a3adaa162634192e71 2024-11-20T05:14:52,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/526d9dcbf4ee4c379469a522ee4df66a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/526d9dcbf4ee4c379469a522ee4df66a 2024-11-20T05:14:52,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ab3b4f07607a40eab5e8a8bb95e67c7e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ab3b4f07607a40eab5e8a8bb95e67c7e 2024-11-20T05:14:52,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9ec155c74f0647e0889f8c135119f22e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9ec155c74f0647e0889f8c135119f22e 2024-11-20T05:14:52,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/58db140f491d416ab2128aa3eb65ed6a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/58db140f491d416ab2128aa3eb65ed6a 2024-11-20T05:14:52,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/aca4870e856043bca99b2144d53efabf to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/aca4870e856043bca99b2144d53efabf 2024-11-20T05:14:52,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/f2ddcc4cd4384a8ea33cd87c32c93d55 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/f2ddcc4cd4384a8ea33cd87c32c93d55 2024-11-20T05:14:52,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ebeb2c1907bf41fb83967be2f4a6f193 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ebeb2c1907bf41fb83967be2f4a6f193 2024-11-20T05:14:52,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9a7018352ce94dfd9535ff7d9075d628 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/9a7018352ce94dfd9535ff7d9075d628 2024-11-20T05:14:52,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c094d85761574ddfb625944cf713e28c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/c094d85761574ddfb625944cf713e28c 2024-11-20T05:14:52,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/93c3ce83370a4024b83ebef6f0b15cdf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2ed50fc10b2644398a5cbc66e572fcd4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/70fdd241071b472eb78d781fd03822a2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bb53a1bc258c43fdbd1b9d1076577730, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/d86f996ed47742348ebec83a66f49149, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc5674b410824171b05aba1116042cae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/f41dacb0cb394ec6a0401a9ff4b32339, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/28d550fe05054c56a8b0244637546cfd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/e466e05655dc43589e6d109780e35cca, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/057aff14a8a34b16940e791b86029ae1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8655a56ba0fd456891d0c339ff5830d1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bceb7de47b464e479a411f6775acdace, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/42b50fa72d3944938e80a72d789d6822, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/0402882bca6b4981abfbf8e1a4a2a25b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2936461edd4649478ac5ea145cdbb6cb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/7bf75f7eef7d41ffa87a73a044b25ac8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8670ba364af049df97736fa49278db71, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/66035bbf8cea4c00870d1d2618717cd0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc88be0e49da46e7939b63a1b2d53341, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/24499f614daa4d6297268d7589da4acb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/264bb33862f94cd18fc708375752ca33, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/979d7de089184172969dc1aa50c890fc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/db26121f7c26463e9ea069e48fad2c7f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/dc98e54e282a41df8c64fd278cc60af6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/be9c12e9b93744a4a98e6d6f6c869b4c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/86099ab8983643cfae4e8c437b259890, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/c76f0e625a364fef9f8d666423199176, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/756f128489074b64821d43ddd4aced08, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/708b2d9809e047ed8669b41da38dce99] to archive 2024-11-20T05:14:52,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:14:52,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/93c3ce83370a4024b83ebef6f0b15cdf to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/93c3ce83370a4024b83ebef6f0b15cdf 2024-11-20T05:14:52,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2ed50fc10b2644398a5cbc66e572fcd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2ed50fc10b2644398a5cbc66e572fcd4 2024-11-20T05:14:52,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/70fdd241071b472eb78d781fd03822a2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/70fdd241071b472eb78d781fd03822a2 2024-11-20T05:14:52,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bb53a1bc258c43fdbd1b9d1076577730 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bb53a1bc258c43fdbd1b9d1076577730 2024-11-20T05:14:52,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/d86f996ed47742348ebec83a66f49149 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/d86f996ed47742348ebec83a66f49149 2024-11-20T05:14:52,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc5674b410824171b05aba1116042cae to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc5674b410824171b05aba1116042cae 2024-11-20T05:14:52,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/f41dacb0cb394ec6a0401a9ff4b32339 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/f41dacb0cb394ec6a0401a9ff4b32339 2024-11-20T05:14:52,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/28d550fe05054c56a8b0244637546cfd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/28d550fe05054c56a8b0244637546cfd 2024-11-20T05:14:52,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/e466e05655dc43589e6d109780e35cca to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/e466e05655dc43589e6d109780e35cca 2024-11-20T05:14:52,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/057aff14a8a34b16940e791b86029ae1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/057aff14a8a34b16940e791b86029ae1 2024-11-20T05:14:52,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8655a56ba0fd456891d0c339ff5830d1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8655a56ba0fd456891d0c339ff5830d1 2024-11-20T05:14:52,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bceb7de47b464e479a411f6775acdace to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/bceb7de47b464e479a411f6775acdace 2024-11-20T05:14:52,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/42b50fa72d3944938e80a72d789d6822 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/42b50fa72d3944938e80a72d789d6822 2024-11-20T05:14:52,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/0402882bca6b4981abfbf8e1a4a2a25b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/0402882bca6b4981abfbf8e1a4a2a25b 2024-11-20T05:14:52,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2936461edd4649478ac5ea145cdbb6cb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/2936461edd4649478ac5ea145cdbb6cb 2024-11-20T05:14:52,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/7bf75f7eef7d41ffa87a73a044b25ac8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/7bf75f7eef7d41ffa87a73a044b25ac8 2024-11-20T05:14:52,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8670ba364af049df97736fa49278db71 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/8670ba364af049df97736fa49278db71 2024-11-20T05:14:52,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/66035bbf8cea4c00870d1d2618717cd0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/66035bbf8cea4c00870d1d2618717cd0 2024-11-20T05:14:52,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc88be0e49da46e7939b63a1b2d53341 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/cc88be0e49da46e7939b63a1b2d53341 2024-11-20T05:14:52,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/24499f614daa4d6297268d7589da4acb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/24499f614daa4d6297268d7589da4acb 2024-11-20T05:14:52,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/264bb33862f94cd18fc708375752ca33 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/264bb33862f94cd18fc708375752ca33 2024-11-20T05:14:52,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/979d7de089184172969dc1aa50c890fc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/979d7de089184172969dc1aa50c890fc 2024-11-20T05:14:52,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/db26121f7c26463e9ea069e48fad2c7f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/db26121f7c26463e9ea069e48fad2c7f 2024-11-20T05:14:52,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/dc98e54e282a41df8c64fd278cc60af6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/dc98e54e282a41df8c64fd278cc60af6 2024-11-20T05:14:52,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/be9c12e9b93744a4a98e6d6f6c869b4c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/be9c12e9b93744a4a98e6d6f6c869b4c 2024-11-20T05:14:52,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/86099ab8983643cfae4e8c437b259890 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/86099ab8983643cfae4e8c437b259890 2024-11-20T05:14:52,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/c76f0e625a364fef9f8d666423199176 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/c76f0e625a364fef9f8d666423199176 2024-11-20T05:14:52,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/756f128489074b64821d43ddd4aced08 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/756f128489074b64821d43ddd4aced08 2024-11-20T05:14:52,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/708b2d9809e047ed8669b41da38dce99 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/708b2d9809e047ed8669b41da38dce99 2024-11-20T05:14:52,575 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/recovered.edits/526.seqid, newMaxSeqId=526, maxSeqId=1 2024-11-20T05:14:52,576 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. 2024-11-20T05:14:52,576 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 098be5f00fb47f2a54b5983aaea0749b: Waiting for close lock at 1732079691128Running coprocessor pre-close hooks at 1732079691128Disabling compacts and flushes for region at 1732079691128Disabling writes for close at 1732079691129 (+1 ms)Obtaining lock to block concurrent updates at 1732079691129Preparing flush snapshotting stores in 098be5f00fb47f2a54b5983aaea0749b at 1732079691129Finished memstore snapshotting TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732079691130 (+1 ms)Flushing stores of TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b. at 1732079691132 (+2 ms)Flushing 098be5f00fb47f2a54b5983aaea0749b/A: creating writer at 1732079691132Flushing 098be5f00fb47f2a54b5983aaea0749b/A: appending metadata at 1732079691140 (+8 ms)Flushing 098be5f00fb47f2a54b5983aaea0749b/A: closing flushed file at 1732079691140Flushing 098be5f00fb47f2a54b5983aaea0749b/B: creating writer at 1732079691584 (+444 ms)Flushing 098be5f00fb47f2a54b5983aaea0749b/B: appending metadata at 1732079691588 (+4 ms)Flushing 098be5f00fb47f2a54b5983aaea0749b/B: closing flushed file at 1732079691588Flushing 098be5f00fb47f2a54b5983aaea0749b/C: creating writer at 1732079692001 (+413 ms)Flushing 098be5f00fb47f2a54b5983aaea0749b/C: appending metadata at 1732079692004 (+3 ms)Flushing 098be5f00fb47f2a54b5983aaea0749b/C: closing flushed file at 1732079692004Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aba76a7: reopening flushed file at 1732079692421 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e9cb031: reopening flushed file at 1732079692430 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75f9cc7d: reopening flushed file at 1732079692436 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 098be5f00fb47f2a54b5983aaea0749b in 1314ms, sequenceid=523, compaction requested=true at 1732079692443 (+7 ms)Writing region close event to WAL at 1732079692571 (+128 ms)Running coprocessor post-close hooks at 1732079692576 (+5 ms)Closed at 1732079692576 2024-11-20T05:14:52,579 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:52,580 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=098be5f00fb47f2a54b5983aaea0749b, regionState=CLOSED 2024-11-20T05:14:52,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:14:52,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-20T05:14:52,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 098be5f00fb47f2a54b5983aaea0749b, server=51d5d2011a1c,44753,1732079655401 in 1.6150 sec 2024-11-20T05:14:52,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-20T05:14:52,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=098be5f00fb47f2a54b5983aaea0749b, UNASSIGN in 1.6240 sec 2024-11-20T05:14:52,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T05:14:52,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6330 sec 2024-11-20T05:14:52,592 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079692591"}]},"ts":"1732079692591"} 2024-11-20T05:14:52,593 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T05:14:52,593 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T05:14:52,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6510 sec 2024-11-20T05:14:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T05:14:53,098 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:53,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T05:14:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:53,114 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T05:14:53,117 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:53,123 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:53,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/recovered.edits] 2024-11-20T05:14:53,133 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3d10d9de13814fcab5f8e4055f7e64c7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/3d10d9de13814fcab5f8e4055f7e64c7 2024-11-20T05:14:53,135 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ac0387241495453bbca3acf4919704c2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/ac0387241495453bbca3acf4919704c2 2024-11-20T05:14:53,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/eda3b86b8dbd4bc5b7dff6d178e8bea9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/A/eda3b86b8dbd4bc5b7dff6d178e8bea9 2024-11-20T05:14:53,140 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/4bc83d38bb614d288d7a413ebf59c573 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/4bc83d38bb614d288d7a413ebf59c573 2024-11-20T05:14:53,142 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ac70431954f245e0a50d9f31b57e6056 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ac70431954f245e0a50d9f31b57e6056 2024-11-20T05:14:53,143 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ed0039a1f6d3401184462ef17a75baf5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/B/ed0039a1f6d3401184462ef17a75baf5 2024-11-20T05:14:53,147 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/10cb2cacaadd431bb0d0ed532afdb55b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/10cb2cacaadd431bb0d0ed532afdb55b 2024-11-20T05:14:53,149 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/49bc8e522ba3444bbbe77a70b01dcba0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/49bc8e522ba3444bbbe77a70b01dcba0 2024-11-20T05:14:53,151 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/5e56c60ffbd24ff69e0d643fa570a72a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/C/5e56c60ffbd24ff69e0d643fa570a72a 2024-11-20T05:14:53,155 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/recovered.edits/526.seqid to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b/recovered.edits/526.seqid 2024-11-20T05:14:53,156 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/098be5f00fb47f2a54b5983aaea0749b 2024-11-20T05:14:53,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T05:14:53,163 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:53,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T05:14:53,175 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T05:14:53,205 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T05:14:53,207 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:53,207 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T05:14:53,207 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732079693207"}]},"ts":"9223372036854775807"} 2024-11-20T05:14:53,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T05:14:53,213 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 098be5f00fb47f2a54b5983aaea0749b, NAME => 'TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T05:14:53,213 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T05:14:53,213 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732079693213"}]},"ts":"9223372036854775807"} 2024-11-20T05:14:53,217 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T05:14:53,218 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:53,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 112 msec 2024-11-20T05:14:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T05:14:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T05:14:53,229 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:53,242 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_142706796_22 at /127.0.0.1:47038 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;51d5d2011a1c:44753-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=430 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4865 (was 5464) 2024-11-20T05:14:53,253 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=430, ProcessCount=11, AvailableMemoryMB=4864 2024-11-20T05:14:53,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:14:53,255 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:14:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:53,258 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T05:14:53,258 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:53,258 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-20T05:14:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T05:14:53,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T05:14:53,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741933_1109 (size=1061) 2024-11-20T05:14:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T05:14:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T05:14:53,673 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:14:53,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741934_1110 (size=53) 2024-11-20T05:14:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T05:14:54,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:54,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d3eebe9eec8dc28c3c19437bf6136526, disabling compactions & flushes 2024-11-20T05:14:54,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. after waiting 0 ms 2024-11-20T05:14:54,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3eebe9eec8dc28c3c19437bf6136526: Waiting for close lock at 1732079694084Disabling compacts and flushes for region at 1732079694084Disabling writes for close at 1732079694085 (+1 ms)Writing region close event to WAL at 1732079694085Closed at 1732079694085 2024-11-20T05:14:54,087 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T05:14:54,088 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732079694087"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732079694087"}]},"ts":"1732079694087"} 2024-11-20T05:14:54,091 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T05:14:54,093 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T05:14:54,094 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079694093"}]},"ts":"1732079694093"} 2024-11-20T05:14:54,097 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T05:14:54,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, ASSIGN}] 2024-11-20T05:14:54,099 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, ASSIGN 2024-11-20T05:14:54,100 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, ASSIGN; state=OFFLINE, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=false 2024-11-20T05:14:54,251 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:54,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, ASSIGN because future has completed 2024-11-20T05:14:54,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:14:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T05:14:54,421 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,421 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:14:54,422 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,422 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:54,422 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,422 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,424 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,426 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:54,427 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3eebe9eec8dc28c3c19437bf6136526 columnFamilyName A 2024-11-20T05:14:54,428 DEBUG [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:54,428 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(327): Store=d3eebe9eec8dc28c3c19437bf6136526/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:54,429 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,430 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:54,431 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3eebe9eec8dc28c3c19437bf6136526 columnFamilyName B 2024-11-20T05:14:54,431 DEBUG [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:54,431 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(327): Store=d3eebe9eec8dc28c3c19437bf6136526/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:54,432 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,433 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:54,433 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3eebe9eec8dc28c3c19437bf6136526 columnFamilyName C 2024-11-20T05:14:54,433 DEBUG [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:54,434 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(327): Store=d3eebe9eec8dc28c3c19437bf6136526/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:54,434 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,434 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,435 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,435 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,437 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,437 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,437 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:14:54,439 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,441 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:14:54,442 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened d3eebe9eec8dc28c3c19437bf6136526; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73305341, jitterRate=0.09233470261096954}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:14:54,442 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:54,443 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for d3eebe9eec8dc28c3c19437bf6136526: Running coprocessor pre-open hook at 1732079694422Writing region info on filesystem at 1732079694422Initializing all the Stores at 1732079694423 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079694424 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079694424Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079694424Cleaning up temporary data from old regions at 1732079694437 (+13 ms)Running coprocessor post-open hooks at 1732079694442 (+5 ms)Region opened successfully at 1732079694443 (+1 ms) 2024-11-20T05:14:54,444 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., pid=32, masterSystemTime=1732079694412 2024-11-20T05:14:54,446 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,446 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:54,447 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=OPEN, openSeqNum=2, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:54,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:14:54,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T05:14:54,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 in 192 msec 2024-11-20T05:14:54,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T05:14:54,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, ASSIGN in 357 msec 2024-11-20T05:14:54,457 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T05:14:54,458 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079694458"}]},"ts":"1732079694458"} 2024-11-20T05:14:54,459 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T05:14:54,460 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T05:14:54,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2060 sec 2024-11-20T05:14:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T05:14:55,408 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:55,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2bb5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:55,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:55,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:55,411 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bb26a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:55,414 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79906bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:55,415 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:55,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:55,420 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T05:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 51d5d2011a1c,37927,1732079654666 2024-11-20T05:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16e27d07 2024-11-20T05:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T05:14:55,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T05:14:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:14:55,430 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:14:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T05:14:55,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741935_1111 (size=1097) 2024-11-20T05:14:55,856 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T05:14:55,856 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T05:14:55,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:14:55,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, REOPEN/MOVE}] 2024-11-20T05:14:55,872 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, REOPEN/MOVE 2024-11-20T05:14:55,873 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:55,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, REOPEN/MOVE because future has completed 2024-11-20T05:14:55,875 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:14:55,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:14:56,029 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,029 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:14:56,029 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing d3eebe9eec8dc28c3c19437bf6136526, disabling compactions & flushes 2024-11-20T05:14:56,030 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,030 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,030 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. after waiting 0 ms 2024-11-20T05:14:56,030 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,037 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T05:14:56,038 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,038 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for d3eebe9eec8dc28c3c19437bf6136526: Waiting for close lock at 1732079696029Running coprocessor pre-close hooks at 1732079696029Disabling compacts and flushes for region at 1732079696029Disabling writes for close at 1732079696030 (+1 ms)Writing region close event to WAL at 1732079696031 (+1 ms)Running coprocessor post-close hooks at 1732079696038 (+7 ms)Closed at 1732079696038 2024-11-20T05:14:56,038 WARN [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: d3eebe9eec8dc28c3c19437bf6136526 to self. 2024-11-20T05:14:56,040 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=CLOSED 2024-11-20T05:14:56,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:14:56,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-20T05:14:56,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 in 168 msec 2024-11-20T05:14:56,046 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, REOPEN/MOVE; state=CLOSED, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=true 2024-11-20T05:14:56,197 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, REOPEN/MOVE because future has completed 2024-11-20T05:14:56,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:14:56,360 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,361 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:14:56,361 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,361 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:14:56,361 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,361 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,366 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,367 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:56,374 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3eebe9eec8dc28c3c19437bf6136526 columnFamilyName A 2024-11-20T05:14:56,376 DEBUG [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:56,377 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(327): Store=d3eebe9eec8dc28c3c19437bf6136526/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:56,377 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,378 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:56,378 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3eebe9eec8dc28c3c19437bf6136526 columnFamilyName B 2024-11-20T05:14:56,378 DEBUG [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:56,379 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(327): Store=d3eebe9eec8dc28c3c19437bf6136526/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:56,379 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,379 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:14:56,380 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3eebe9eec8dc28c3c19437bf6136526 columnFamilyName C 2024-11-20T05:14:56,380 DEBUG [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:56,380 INFO [StoreOpener-d3eebe9eec8dc28c3c19437bf6136526-1 {}] regionserver.HStore(327): Store=d3eebe9eec8dc28c3c19437bf6136526/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:14:56,380 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,380 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,381 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,382 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,383 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,383 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,384 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:14:56,385 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,386 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened d3eebe9eec8dc28c3c19437bf6136526; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60261261, jitterRate=-0.10203723609447479}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:14:56,386 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,388 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for d3eebe9eec8dc28c3c19437bf6136526: Running coprocessor pre-open hook at 1732079696361Writing region info on filesystem at 1732079696361Initializing all the Stores at 1732079696362 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079696362Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079696363 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079696363Cleaning up temporary data from old regions at 1732079696383 (+20 ms)Running coprocessor post-open hooks at 1732079696386 (+3 ms)Region opened successfully at 1732079696388 (+2 ms) 2024-11-20T05:14:56,389 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., pid=37, masterSystemTime=1732079696357 2024-11-20T05:14:56,391 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,391 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,391 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=OPEN, openSeqNum=5, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:14:56,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-20T05:14:56,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 in 191 msec 2024-11-20T05:14:56,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-20T05:14:56,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, REOPEN/MOVE in 525 msec 2024-11-20T05:14:56,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-20T05:14:56,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-11-20T05:14:56,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 972 msec 2024-11-20T05:14:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T05:14:56,407 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:14:56,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01494b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0d9701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3310dfa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28944b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,413 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be3d499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa3a6f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95d199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,416 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c153fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,418 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1631d87c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e51468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,420 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf30124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,421 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3c0c6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1fcab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,423 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b48c26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,425 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3368e8f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337f1b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,427 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5498d37e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,428 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b829578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62885e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683de4da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c58b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138dc9c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721efe8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c264dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:14:56,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:14:56,437 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc93dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:14:56,439 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:14:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c31ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:14:56,441 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,441 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,441 DEBUG [Thread-580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,441 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,441 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,441 DEBUG [Thread-584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,441 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:14:56,441 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,442 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-20T05:14:56,443 DEBUG [Thread-591 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,443 DEBUG [Thread-582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:14:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,444 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:14:56,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:14:56,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,446 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:14:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:14:56,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:14:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:14:56,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:14:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:14:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:14:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:14:56,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:56,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:14:56,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:56,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:14:56,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:56,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207d111aec10214d239a30afe36c388942_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079696466/Put/seqid=0 2024-11-20T05:14:56,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732079696500, max timestamp=1732079696512Num uniques 859; 2024-11-20T05:14:56,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732079696500, max timestamp=1732079696512Num uniques 859; 2024-11-20T05:14:56,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T05:14:56,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T05:14:56,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732079696500, max timestamp=1732079696512Num uniques 859; 2024-11-20T05:14:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079756509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T05:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741936_1112 (size=14594) 2024-11-20T05:14:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:14:56,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079756550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079756550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079756552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079756557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079756627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079756667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079756667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079756668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079756668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:56,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:56,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:14:56,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079756838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079756877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079756878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079756878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079756878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:56,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:56,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:56,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:56,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:56,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:56,954 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207d111aec10214d239a30afe36c388942_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207d111aec10214d239a30afe36c388942_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:56,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/226632a2623f4ca8bf0d5a0cec2f55f7, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:14:56,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/226632a2623f4ca8bf0d5a0cec2f55f7 is 175, key is test_row_0/A:col10/1732079696466/Put/seqid=0 2024-11-20T05:14:56,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741937_1113 (size=39549) 2024-11-20T05:14:56,983 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/226632a2623f4ca8bf0d5a0cec2f55f7 2024-11-20T05:14:57,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/256637ed4af043cd8b84c1fac618eb30 is 50, key is test_row_0/B:col10/1732079696466/Put/seqid=0 2024-11-20T05:14:57,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741938_1114 (size=12001) 2024-11-20T05:14:57,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:57,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:57,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:14:57,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079757147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079757187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079757188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079757188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079757188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:57,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:57,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/256637ed4af043cd8b84c1fac618eb30 2024-11-20T05:14:57,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/68318fa6820b43df823ebe8c5693cbf6 is 50, key is test_row_0/C:col10/1732079696466/Put/seqid=0 2024-11-20T05:14:57,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741939_1115 (size=12001) 2024-11-20T05:14:57,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/68318fa6820b43df823ebe8c5693cbf6 2024-11-20T05:14:57,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/226632a2623f4ca8bf0d5a0cec2f55f7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/226632a2623f4ca8bf0d5a0cec2f55f7 2024-11-20T05:14:57,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/226632a2623f4ca8bf0d5a0cec2f55f7, entries=200, sequenceid=14, filesize=38.6 K 2024-11-20T05:14:57,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/256637ed4af043cd8b84c1fac618eb30 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/256637ed4af043cd8b84c1fac618eb30 2024-11-20T05:14:57,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/256637ed4af043cd8b84c1fac618eb30, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T05:14:57,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/68318fa6820b43df823ebe8c5693cbf6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/68318fa6820b43df823ebe8c5693cbf6 2024-11-20T05:14:57,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/68318fa6820b43df823ebe8c5693cbf6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T05:14:57,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:57,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for d3eebe9eec8dc28c3c19437bf6136526 in 1055ms, sequenceid=14, compaction requested=false 2024-11-20T05:14:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,522 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T05:14:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:14:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:14:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:57,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-20T05:14:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:14:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:14:57,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:57,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:57,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112027fd68f5dbca4eaba0d1b3dd22f7f771_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079696554/Put/seqid=0 2024-11-20T05:14:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741940_1116 (size=14594) 2024-11-20T05:14:57,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079757698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079757698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079757705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079757708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079757708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079757817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079757817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079757828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:57,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:57,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:57,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:57,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:57,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:57,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079758027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079758027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079758037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:58,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112027fd68f5dbca4eaba0d1b3dd22f7f771_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027fd68f5dbca4eaba0d1b3dd22f7f771_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:58,110 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/084ffed184f041cca62fea8c7e76cefd, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:14:58,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/084ffed184f041cca62fea8c7e76cefd is 175, key is test_row_0/A:col10/1732079696554/Put/seqid=0 2024-11-20T05:14:58,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741941_1117 (size=39549) 2024-11-20T05:14:58,119 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T05:14:58,123 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/084ffed184f041cca62fea8c7e76cefd 2024-11-20T05:14:58,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/86aef5b96f624ebdbfd0a129f98c5542 is 50, key is test_row_0/B:col10/1732079696554/Put/seqid=0 2024-11-20T05:14:58,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741942_1118 (size=12001) 2024-11-20T05:14:58,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/86aef5b96f624ebdbfd0a129f98c5542 2024-11-20T05:14:58,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/8f8af35bffbd46b8bd5aadcb6242c354 is 50, key is test_row_0/C:col10/1732079696554/Put/seqid=0 2024-11-20T05:14:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741943_1119 (size=12001) 2024-11-20T05:14:58,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:58,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079758337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079758337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079758347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:58,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,577 INFO [master/51d5d2011a1c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T05:14:58,577 INFO [master/51d5d2011a1c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T05:14:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:14:58,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/8f8af35bffbd46b8bd5aadcb6242c354 2024-11-20T05:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/084ffed184f041cca62fea8c7e76cefd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/084ffed184f041cca62fea8c7e76cefd 2024-11-20T05:14:58,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/084ffed184f041cca62fea8c7e76cefd, entries=200, sequenceid=50, filesize=38.6 K 2024-11-20T05:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/86aef5b96f624ebdbfd0a129f98c5542 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/86aef5b96f624ebdbfd0a129f98c5542 2024-11-20T05:14:58,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/86aef5b96f624ebdbfd0a129f98c5542, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T05:14:58,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/8f8af35bffbd46b8bd5aadcb6242c354 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/8f8af35bffbd46b8bd5aadcb6242c354 2024-11-20T05:14:58,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/8f8af35bffbd46b8bd5aadcb6242c354, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T05:14:58,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for d3eebe9eec8dc28c3c19437bf6136526 in 956ms, sequenceid=50, compaction requested=false 2024-11-20T05:14:58,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:14:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:58,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:14:58,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:14:58,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:58,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:14:58,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:58,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:14:58,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:14:58,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a3ca763893b94a39aa7be089a54f7e6f_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079698718/Put/seqid=0 2024-11-20T05:14:58,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741944_1120 (size=14594) 2024-11-20T05:14:58,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:58,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:58,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079758780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079698758, max timestamp=1732079698784Num uniques 859; 2024-11-20T05:14:58,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079698758, max timestamp=1732079698784Num uniques 859; 2024-11-20T05:14:58,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:14:58,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079698758, max timestamp=1732079698784Num uniques 859; 2024-11-20T05:14:58,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:14:58,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:14:58,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079758852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079758853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079758858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079758858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079758897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:58,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:58,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079758967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079758967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:58,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079758977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079759107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,153 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:14:59,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a3ca763893b94a39aa7be089a54f7e6f_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a3ca763893b94a39aa7be089a54f7e6f_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:14:59,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:14:59,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5 is 175, key is test_row_0/A:col10/1732079698718/Put/seqid=0 2024-11-20T05:14:59,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741945_1121 (size=39549) 2024-11-20T05:14:59,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079759177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079759177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079759187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:59,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:59,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079759417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079759487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079759487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079759507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:59,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:59,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5 2024-11-20T05:14:59,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/b4321c1757ed45f6964581a5f50f73ff is 50, key is test_row_0/B:col10/1732079698718/Put/seqid=0 2024-11-20T05:14:59,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741946_1122 (size=12001) 2024-11-20T05:14:59,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:59,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:59,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079759887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,891 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:14:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:14:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079759927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:14:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:14:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:14:59,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/b4321c1757ed45f6964581a5f50f73ff 2024-11-20T05:14:59,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:14:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:14:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:14:59,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:14:59,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/a024a757f37d4fa78871b455dfba457e is 50, key is test_row_0/C:col10/1732079698718/Put/seqid=0 2024-11-20T05:14:59,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741947_1123 (size=12001) 2024-11-20T05:15:00,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079759997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079759997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:00,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079760017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:00,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:15:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:00,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:00,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/a024a757f37d4fa78871b455dfba457e 2024-11-20T05:15:00,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5 2024-11-20T05:15:00,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5, entries=200, sequenceid=61, filesize=38.6 K 2024-11-20T05:15:00,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/b4321c1757ed45f6964581a5f50f73ff as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/b4321c1757ed45f6964581a5f50f73ff 2024-11-20T05:15:00,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/b4321c1757ed45f6964581a5f50f73ff, entries=150, sequenceid=61, filesize=11.7 K 2024-11-20T05:15:00,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/a024a757f37d4fa78871b455dfba457e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a024a757f37d4fa78871b455dfba457e 2024-11-20T05:15:00,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a024a757f37d4fa78871b455dfba457e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-20T05:15:00,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d3eebe9eec8dc28c3c19437bf6136526 in 1716ms, sequenceid=61, compaction requested=true 2024-11-20T05:15:00,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:00,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:00,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:00,436 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:00,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:00,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:00,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:00,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T05:15:00,437 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118647 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:00,437 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/A is initiating minor compaction (all files) 2024-11-20T05:15:00,437 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/A in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,437 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/226632a2623f4ca8bf0d5a0cec2f55f7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/084ffed184f041cca62fea8c7e76cefd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=115.9 K 2024-11-20T05:15:00,437 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,438 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/226632a2623f4ca8bf0d5a0cec2f55f7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/084ffed184f041cca62fea8c7e76cefd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5] 2024-11-20T05:15:00,438 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226632a2623f4ca8bf0d5a0cec2f55f7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732079696455 2024-11-20T05:15:00,439 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084ffed184f041cca62fea8c7e76cefd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732079696537 2024-11-20T05:15:00,439 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c6bc95f0ad4dcfb1afc7d6216f86c5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732079697690 2024-11-20T05:15:00,440 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:00,442 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:00,442 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/B is initiating minor compaction (all files) 2024-11-20T05:15:00,442 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/B in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,442 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/256637ed4af043cd8b84c1fac618eb30, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/86aef5b96f624ebdbfd0a129f98c5542, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/b4321c1757ed45f6964581a5f50f73ff] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=35.2 K 2024-11-20T05:15:00,443 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 256637ed4af043cd8b84c1fac618eb30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732079696455 2024-11-20T05:15:00,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T05:15:00,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,445 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-20T05:15:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:00,446 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 86aef5b96f624ebdbfd0a129f98c5542, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732079696547 2024-11-20T05:15:00,447 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b4321c1757ed45f6964581a5f50f73ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732079697690 2024-11-20T05:15:00,475 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:00,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba3129ed17a540829b14a98f22cd7de8_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079698852/Put/seqid=0 2024-11-20T05:15:00,491 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411208e6d8e0be1fe4b1892f29a0da523f98e_d3eebe9eec8dc28c3c19437bf6136526 store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:00,494 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#B#compaction#107 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:00,494 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/8f5260ada68b4be7bc312477c4c64ce5 is 50, key is test_row_0/B:col10/1732079698718/Put/seqid=0 2024-11-20T05:15:00,502 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411208e6d8e0be1fe4b1892f29a0da523f98e_d3eebe9eec8dc28c3c19437bf6136526, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:00,503 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208e6d8e0be1fe4b1892f29a0da523f98e_d3eebe9eec8dc28c3c19437bf6136526 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:00,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741949_1125 (size=12104) 2024-11-20T05:15:00,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741948_1124 (size=12154) 2024-11-20T05:15:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:00,574 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba3129ed17a540829b14a98f22cd7de8_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba3129ed17a540829b14a98f22cd7de8_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:00,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741950_1126 (size=4469) 2024-11-20T05:15:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/1cbcc359335f4ef691d071dd63e93a76, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:00,580 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#A#compaction#105 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:00,581 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/3c6863b8e65c4f708d0e09895f71f526 is 175, key is test_row_0/A:col10/1732079698718/Put/seqid=0 2024-11-20T05:15:00,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/1cbcc359335f4ef691d071dd63e93a76 is 175, key is test_row_0/A:col10/1732079698852/Put/seqid=0 2024-11-20T05:15:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:15:00,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741951_1127 (size=31058) 2024-11-20T05:15:00,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741952_1128 (size=30955) 2024-11-20T05:15:00,646 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/1cbcc359335f4ef691d071dd63e93a76 2024-11-20T05:15:00,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0faeef0fbb0b499f99537ec087f36501 is 50, key is test_row_0/B:col10/1732079698852/Put/seqid=0 2024-11-20T05:15:00,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741953_1129 (size=12001) 2024-11-20T05:15:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:00,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:00,948 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/8f5260ada68b4be7bc312477c4c64ce5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/8f5260ada68b4be7bc312477c4c64ce5 2024-11-20T05:15:00,955 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/B of d3eebe9eec8dc28c3c19437bf6136526 into 8f5260ada68b4be7bc312477c4c64ce5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:00,955 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:00,955 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/B, priority=13, startTime=1732079700436; duration=0sec 2024-11-20T05:15:00,955 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:00,955 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:B 2024-11-20T05:15:00,955 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:00,957 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:00,957 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/C is initiating minor compaction (all files) 2024-11-20T05:15:00,957 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/C in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:00,957 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/68318fa6820b43df823ebe8c5693cbf6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/8f8af35bffbd46b8bd5aadcb6242c354, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a024a757f37d4fa78871b455dfba457e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=35.2 K 2024-11-20T05:15:00,958 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 68318fa6820b43df823ebe8c5693cbf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732079696455 2024-11-20T05:15:00,958 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8af35bffbd46b8bd5aadcb6242c354, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732079696547 2024-11-20T05:15:00,958 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a024a757f37d4fa78871b455dfba457e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732079697690 2024-11-20T05:15:00,976 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#C#compaction#109 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:00,977 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/5872ea8308c9439888235a6b583e6866 is 50, key is test_row_0/C:col10/1732079698718/Put/seqid=0 2024-11-20T05:15:00,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741954_1130 (size=12104) 2024-11-20T05:15:01,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079760993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:01,005 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/3c6863b8e65c4f708d0e09895f71f526 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/3c6863b8e65c4f708d0e09895f71f526 2024-11-20T05:15:01,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079761008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:01,009 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:01,014 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/A of d3eebe9eec8dc28c3c19437bf6136526 into 3c6863b8e65c4f708d0e09895f71f526(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:01,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:01,014 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/A, priority=13, startTime=1732079700436; duration=0sec 2024-11-20T05:15:01,015 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:01,015 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:A 2024-11-20T05:15:01,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079761017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:01,025 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:01,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079761037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:01,044 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:01,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079761107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:01,112 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0faeef0fbb0b499f99537ec087f36501 2024-11-20T05:15:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/53ba64d696444020999e50e1406f840d is 50, key is test_row_0/C:col10/1732079698852/Put/seqid=0 2024-11-20T05:15:01,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741955_1131 (size=12001) 2024-11-20T05:15:01,141 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/53ba64d696444020999e50e1406f840d 2024-11-20T05:15:01,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/1cbcc359335f4ef691d071dd63e93a76 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/1cbcc359335f4ef691d071dd63e93a76 2024-11-20T05:15:01,155 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/1cbcc359335f4ef691d071dd63e93a76, entries=150, sequenceid=95, filesize=30.2 K 2024-11-20T05:15:01,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0faeef0fbb0b499f99537ec087f36501 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0faeef0fbb0b499f99537ec087f36501 2024-11-20T05:15:01,164 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0faeef0fbb0b499f99537ec087f36501, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T05:15:01,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/53ba64d696444020999e50e1406f840d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/53ba64d696444020999e50e1406f840d 2024-11-20T05:15:01,171 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/53ba64d696444020999e50e1406f840d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T05:15:01,173 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for d3eebe9eec8dc28c3c19437bf6136526 in 727ms, sequenceid=95, compaction requested=false 2024-11-20T05:15:01,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:01,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:01,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-20T05:15:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-20T05:15:01,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-20T05:15:01,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.7280 sec 2024-11-20T05:15:01,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 4.7380 sec 2024-11-20T05:15:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:01,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T05:15:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:01,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a4e3a4597e074ca2b188a6899d404238_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079701319/Put/seqid=0 2024-11-20T05:15:01,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741956_1132 (size=14594) 2024-11-20T05:15:01,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:01,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a4e3a4597e074ca2b188a6899d404238_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a4e3a4597e074ca2b188a6899d404238_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:01,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/a51e0414b8614f5ba54a06c32135346d, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:01,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/a51e0414b8614f5ba54a06c32135346d is 175, key is test_row_0/A:col10/1732079701319/Put/seqid=0 2024-11-20T05:15:01,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741957_1133 (size=39549) 2024-11-20T05:15:01,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/a51e0414b8614f5ba54a06c32135346d 2024-11-20T05:15:01,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0e2e594681334bfe8d01df212558f1f3 is 50, key is test_row_0/B:col10/1732079701319/Put/seqid=0 2024-11-20T05:15:01,405 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/5872ea8308c9439888235a6b583e6866 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/5872ea8308c9439888235a6b583e6866 2024-11-20T05:15:01,419 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/C of d3eebe9eec8dc28c3c19437bf6136526 into 5872ea8308c9439888235a6b583e6866(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:01,419 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:01,420 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/C, priority=13, startTime=1732079700436; duration=0sec 2024-11-20T05:15:01,420 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:01,420 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:C 2024-11-20T05:15:01,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079761434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:01,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741958_1134 (size=12001) 2024-11-20T05:15:01,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0e2e594681334bfe8d01df212558f1f3 2024-11-20T05:15:01,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/cf088029e7bc4724b14f0af79371bebc is 50, key is test_row_0/C:col10/1732079701319/Put/seqid=0 2024-11-20T05:15:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741959_1135 (size=12001) 2024-11-20T05:15:01,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/cf088029e7bc4724b14f0af79371bebc 2024-11-20T05:15:01,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/a51e0414b8614f5ba54a06c32135346d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/a51e0414b8614f5ba54a06c32135346d 2024-11-20T05:15:01,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/a51e0414b8614f5ba54a06c32135346d, entries=200, sequenceid=109, filesize=38.6 K 2024-11-20T05:15:01,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0e2e594681334bfe8d01df212558f1f3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0e2e594681334bfe8d01df212558f1f3 2024-11-20T05:15:01,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0e2e594681334bfe8d01df212558f1f3, entries=150, sequenceid=109, filesize=11.7 K 2024-11-20T05:15:01,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/cf088029e7bc4724b14f0af79371bebc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/cf088029e7bc4724b14f0af79371bebc 2024-11-20T05:15:01,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/cf088029e7bc4724b14f0af79371bebc, entries=150, sequenceid=109, filesize=11.7 K 2024-11-20T05:15:01,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d3eebe9eec8dc28c3c19437bf6136526 in 193ms, sequenceid=109, compaction requested=true 2024-11-20T05:15:01,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:01,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:01,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:01,513 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:01,513 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:01,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:01,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:01,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:01,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:01,515 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:01,515 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/A is initiating minor compaction (all files) 2024-11-20T05:15:01,515 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/A in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:01,515 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/3c6863b8e65c4f708d0e09895f71f526, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/1cbcc359335f4ef691d071dd63e93a76, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/a51e0414b8614f5ba54a06c32135346d] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=99.2 K 2024-11-20T05:15:01,515 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:01,515 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/3c6863b8e65c4f708d0e09895f71f526, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/1cbcc359335f4ef691d071dd63e93a76, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/a51e0414b8614f5ba54a06c32135346d] 2024-11-20T05:15:01,515 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:01,515 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/B is initiating minor compaction (all files) 2024-11-20T05:15:01,516 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/B in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:01,516 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/8f5260ada68b4be7bc312477c4c64ce5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0faeef0fbb0b499f99537ec087f36501, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0e2e594681334bfe8d01df212558f1f3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=35.3 K 2024-11-20T05:15:01,516 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c6863b8e65c4f708d0e09895f71f526, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732079697690 2024-11-20T05:15:01,517 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5260ada68b4be7bc312477c4c64ce5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732079697690 2024-11-20T05:15:01,517 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cbcc359335f4ef691d071dd63e93a76, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079698850 2024-11-20T05:15:01,518 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0faeef0fbb0b499f99537ec087f36501, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079698850 2024-11-20T05:15:01,520 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a51e0414b8614f5ba54a06c32135346d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732079700971 2024-11-20T05:15:01,520 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e2e594681334bfe8d01df212558f1f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732079700971 2024-11-20T05:15:01,553 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:01,562 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:01,563 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0f86f1b0b6d64b7e83b95ae8c12178f8 is 50, key is test_row_0/B:col10/1732079701319/Put/seqid=0 2024-11-20T05:15:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:01,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079701411, max timestamp=1732079701551Num uniques 859; 2024-11-20T05:15:01,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:01,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079701411, max timestamp=1732079701551Num uniques 859; 2024-11-20T05:15:01,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:01,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079701411, max timestamp=1732079701551Num uniques 859; 2024-11-20T05:15:01,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-20T05:15:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:01,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:01,621 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411206a5e485a978348d9b3bd586916ea2f6c_d3eebe9eec8dc28c3c19437bf6136526 store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:01,624 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411206a5e485a978348d9b3bd586916ea2f6c_d3eebe9eec8dc28c3c19437bf6136526, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:01,624 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206a5e485a978348d9b3bd586916ea2f6c_d3eebe9eec8dc28c3c19437bf6136526 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:01,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741960_1136 (size=12207) 2024-11-20T05:15:01,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112090b649e80108476d9ee7a3d31243c9d8_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079701411/Put/seqid=0 2024-11-20T05:15:01,672 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/0f86f1b0b6d64b7e83b95ae8c12178f8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0f86f1b0b6d64b7e83b95ae8c12178f8 2024-11-20T05:15:01,682 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/B of d3eebe9eec8dc28c3c19437bf6136526 into 0f86f1b0b6d64b7e83b95ae8c12178f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:01,682 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:01,682 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/B, priority=13, startTime=1732079701513; duration=0sec 2024-11-20T05:15:01,682 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:01,682 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:B 2024-11-20T05:15:01,682 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:01,685 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:01,685 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/C is initiating minor compaction (all files) 2024-11-20T05:15:01,685 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/C in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:01,686 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/5872ea8308c9439888235a6b583e6866, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/53ba64d696444020999e50e1406f840d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/cf088029e7bc4724b14f0af79371bebc] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=35.3 K 2024-11-20T05:15:01,686 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5872ea8308c9439888235a6b583e6866, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732079697690 2024-11-20T05:15:01,687 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ba64d696444020999e50e1406f840d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079698850 2024-11-20T05:15:01,687 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting cf088029e7bc4724b14f0af79371bebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732079700971 2024-11-20T05:15:01,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741961_1137 (size=4469) 2024-11-20T05:15:01,715 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:01,716 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/4d08f532f1124a9181881354131dd020 is 50, key is test_row_0/C:col10/1732079701319/Put/seqid=0 2024-11-20T05:15:01,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741962_1138 (size=12204) 2024-11-20T05:15:01,737 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:01,741 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112090b649e80108476d9ee7a3d31243c9d8_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112090b649e80108476d9ee7a3d31243c9d8_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:01,742 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e0e1a8486b7c4500a2d56212227aad5a, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:01,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e0e1a8486b7c4500a2d56212227aad5a is 175, key is test_row_0/A:col10/1732079701411/Put/seqid=0 2024-11-20T05:15:01,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741963_1139 (size=12207) 2024-11-20T05:15:01,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079701723, max timestamp=1732079701755Num uniques 859; 2024-11-20T05:15:01,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:01,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079701723, max timestamp=1732079701755Num uniques 859; 2024-11-20T05:15:01,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:01,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079701723, max timestamp=1732079701755Num uniques 859; 2024-11-20T05:15:01,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:01,763 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/4d08f532f1124a9181881354131dd020 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/4d08f532f1124a9181881354131dd020 2024-11-20T05:15:01,777 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/C of d3eebe9eec8dc28c3c19437bf6136526 into 4d08f532f1124a9181881354131dd020(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:01,778 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:01,778 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/C, priority=13, startTime=1732079701514; duration=0sec 2024-11-20T05:15:01,778 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:01,778 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:C 2024-11-20T05:15:01,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741964_1140 (size=31005) 2024-11-20T05:15:01,783 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e0e1a8486b7c4500a2d56212227aad5a 2024-11-20T05:15:01,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/e5e31580a26e43ada7e6b2f11425058f is 50, key is test_row_0/B:col10/1732079701411/Put/seqid=0 2024-11-20T05:15:01,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741965_1141 (size=12051) 2024-11-20T05:15:01,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/e5e31580a26e43ada7e6b2f11425058f 2024-11-20T05:15:01,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/86f78494227d4f1982e1ae97a3b956aa is 50, key is test_row_0/C:col10/1732079701411/Put/seqid=0 2024-11-20T05:15:01,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741966_1142 (size=12051) 2024-11-20T05:15:01,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079701841, max timestamp=1732079701855Num uniques 1018; 2024-11-20T05:15:01,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079701841, max timestamp=1732079701855Num uniques 1018; 2024-11-20T05:15:01,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079701841, max timestamp=1732079701855Num uniques 1018; 2024-11-20T05:15:01,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:15:01,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:15:01,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:15:01,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732079701906, max timestamp=1732079701924Num uniques 1027; 2024-11-20T05:15:01,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-20T05:15:01,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12672, totalHeapSize=29048, min timestamp=1732079701906, max timestamp=1732079701924Num uniques 1027; 2024-11-20T05:15:01,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12672, totalHeapSize=29048, min timestamp=1732079701906, max timestamp=1732079701924Num uniques 1027; 2024-11-20T05:15:01,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47022, new segment data size=12672, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-20T05:15:01,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47022, new segment data size=12672, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-20T05:15:01,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732079701969, max timestamp=1732079701981Num uniques 1136; 2024-11-20T05:15:01,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-20T05:15:01,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732079701969, max timestamp=1732079701981Num uniques 1136; 2024-11-20T05:15:01,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732079701969, max timestamp=1732079701981Num uniques 1136; 2024-11-20T05:15:01,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-20T05:15:01,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-20T05:15:02,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079702013, max timestamp=1732079702044Num uniques 1095; 2024-11-20T05:15:02,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-20T05:15:02,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079702013, max timestamp=1732079702044Num uniques 1095; 2024-11-20T05:15:02,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-20T05:15:02,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079702013, max timestamp=1732079702044Num uniques 1095; 2024-11-20T05:15:02,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-20T05:15:02,110 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#A#compaction#114 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:02,111 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/f9a2fd2f99b84a91af919245c9f3b1bd is 175, key is test_row_0/A:col10/1732079701319/Put/seqid=0 2024-11-20T05:15:02,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741967_1143 (size=31161) 2024-11-20T05:15:02,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732079702106, max timestamp=1732079702134Num uniques 1104; 2024-11-20T05:15:02,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-20T05:15:02,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732079702115, max timestamp=1732079702134Num uniques 1104; 2024-11-20T05:15:02,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-20T05:15:02,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732079702106, max timestamp=1732079702134Num uniques 1104; 2024-11-20T05:15:02,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-20T05:15:02,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732079702124, max timestamp=1732079702179Num uniques 1113; 2024-11-20T05:15:02,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732079702124, max timestamp=1732079702179Num uniques 1113; 2024-11-20T05:15:02,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-20T05:15:02,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-20T05:15:02,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732079702124, max timestamp=1732079702179Num uniques 1013; 2024-11-20T05:15:02,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-20T05:15:02,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/86f78494227d4f1982e1ae97a3b956aa 2024-11-20T05:15:02,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e0e1a8486b7c4500a2d56212227aad5a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e0e1a8486b7c4500a2d56212227aad5a 2024-11-20T05:15:02,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732079702219, max timestamp=1732079702237Num uniques 1072; 2024-11-20T05:15:02,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-20T05:15:02,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732079702219, max timestamp=1732079702237Num uniques 1072; 2024-11-20T05:15:02,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-20T05:15:02,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732079702219, max timestamp=1732079702237Num uniques 1072; 2024-11-20T05:15:02,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-20T05:15:02,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e0e1a8486b7c4500a2d56212227aad5a, entries=150, sequenceid=131, filesize=30.3 K 2024-11-20T05:15:02,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/e5e31580a26e43ada7e6b2f11425058f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e5e31580a26e43ada7e6b2f11425058f 2024-11-20T05:15:02,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e5e31580a26e43ada7e6b2f11425058f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-20T05:15:02,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/86f78494227d4f1982e1ae97a3b956aa as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/86f78494227d4f1982e1ae97a3b956aa 2024-11-20T05:15:02,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/86f78494227d4f1982e1ae97a3b956aa, entries=150, sequenceid=131, filesize=11.8 K 2024-11-20T05:15:02,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=80.51 KB/82440 for d3eebe9eec8dc28c3c19437bf6136526 in 654ms, sequenceid=131, compaction requested=false 2024-11-20T05:15:02,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:02,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=87.22 KB heapSize=234.94 KB 2024-11-20T05:15:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:02,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a6cb2e74c12c4c01ab8be6017efcc6a4_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079702269/Put/seqid=0 2024-11-20T05:15:02,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741968_1144 (size=17534) 2024-11-20T05:15:02,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079762317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079762317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:02,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079762427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079762427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:02,538 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/f9a2fd2f99b84a91af919245c9f3b1bd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/f9a2fd2f99b84a91af919245c9f3b1bd 2024-11-20T05:15:02,543 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/A of d3eebe9eec8dc28c3c19437bf6136526 into f9a2fd2f99b84a91af919245c9f3b1bd(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-20T05:15:02,544 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:02,544 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/A, priority=13, startTime=1732079701513; duration=1sec 2024-11-20T05:15:02,544 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:02,544 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:A 2024-11-20T05:15:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079762637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079762637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:02,699 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:02,703 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a6cb2e74c12c4c01ab8be6017efcc6a4_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6cb2e74c12c4c01ab8be6017efcc6a4_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:02,704 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c6ed69e43ac74148a030ff05ae3500bd, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:02,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c6ed69e43ac74148a030ff05ae3500bd is 175, key is test_row_0/A:col10/1732079702269/Put/seqid=0 2024-11-20T05:15:02,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741969_1145 (size=48639) 2024-11-20T05:15:02,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079762947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079762947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:03,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079763027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:03,028 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:03,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079763047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:03,052 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:03,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079763058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:03,060 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:03,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c6ed69e43ac74148a030ff05ae3500bd 2024-11-20T05:15:03,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/073284b09db347d7a43ac885dbe541bc is 50, key is test_row_0/B:col10/1732079702269/Put/seqid=0 2024-11-20T05:15:03,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741970_1146 (size=12301) 2024-11-20T05:15:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079763457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079763457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:03,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/073284b09db347d7a43ac885dbe541bc 2024-11-20T05:15:03,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/89e1b62d0185434f9e0fa2eb425c4c7c is 50, key is test_row_0/C:col10/1732079702269/Put/seqid=0 2024-11-20T05:15:03,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741971_1147 (size=12301) 2024-11-20T05:15:03,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/89e1b62d0185434f9e0fa2eb425c4c7c 2024-11-20T05:15:03,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/c6ed69e43ac74148a030ff05ae3500bd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c6ed69e43ac74148a030ff05ae3500bd 2024-11-20T05:15:03,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c6ed69e43ac74148a030ff05ae3500bd, entries=250, sequenceid=281, filesize=47.5 K 2024-11-20T05:15:03,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/073284b09db347d7a43ac885dbe541bc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/073284b09db347d7a43ac885dbe541bc 2024-11-20T05:15:03,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/073284b09db347d7a43ac885dbe541bc, entries=150, sequenceid=281, filesize=12.0 K 2024-11-20T05:15:03,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/89e1b62d0185434f9e0fa2eb425c4c7c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/89e1b62d0185434f9e0fa2eb425c4c7c 2024-11-20T05:15:03,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/89e1b62d0185434f9e0fa2eb425c4c7c, entries=150, sequenceid=281, filesize=12.0 K 2024-11-20T05:15:03,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~234.89 KB/240528, currentSize=100.63 KB/103050 for d3eebe9eec8dc28c3c19437bf6136526 in 1687ms, sequenceid=281, compaction requested=true 2024-11-20T05:15:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:03,959 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:03,959 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:03,960 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:03,960 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36559 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:03,960 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/A is initiating minor compaction (all files) 2024-11-20T05:15:03,960 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/B is initiating minor compaction (all files) 2024-11-20T05:15:03,960 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/B in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:03,960 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/A in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:03,960 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/f9a2fd2f99b84a91af919245c9f3b1bd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e0e1a8486b7c4500a2d56212227aad5a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c6ed69e43ac74148a030ff05ae3500bd] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=108.2 K 2024-11-20T05:15:03,960 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0f86f1b0b6d64b7e83b95ae8c12178f8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e5e31580a26e43ada7e6b2f11425058f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/073284b09db347d7a43ac885dbe541bc] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=35.7 K 2024-11-20T05:15:03,960 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:03,960 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/f9a2fd2f99b84a91af919245c9f3b1bd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e0e1a8486b7c4500a2d56212227aad5a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c6ed69e43ac74148a030ff05ae3500bd] 2024-11-20T05:15:03,961 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f86f1b0b6d64b7e83b95ae8c12178f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732079700971 2024-11-20T05:15:03,961 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a2fd2f99b84a91af919245c9f3b1bd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732079700971 2024-11-20T05:15:03,961 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e5e31580a26e43ada7e6b2f11425058f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732079701411 2024-11-20T05:15:03,961 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e1a8486b7c4500a2d56212227aad5a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732079701411 2024-11-20T05:15:03,961 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 073284b09db347d7a43ac885dbe541bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732079702247 2024-11-20T05:15:03,962 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6ed69e43ac74148a030ff05ae3500bd, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732079702247 2024-11-20T05:15:03,977 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:03,979 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:03,980 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/53ead9c628aa4d9891fba5ea9279e49e is 50, key is test_row_0/B:col10/1732079702269/Put/seqid=0 2024-11-20T05:15:03,993 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112033e5ba7f541744caa531b8c63842d254_d3eebe9eec8dc28c3c19437bf6136526 store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:03,995 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112033e5ba7f541744caa531b8c63842d254_d3eebe9eec8dc28c3c19437bf6136526, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:03,996 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112033e5ba7f541744caa531b8c63842d254_d3eebe9eec8dc28c3c19437bf6136526 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:04,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741972_1148 (size=12609) 2024-11-20T05:15:04,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741973_1149 (size=4469) 2024-11-20T05:15:04,408 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/53ead9c628aa4d9891fba5ea9279e49e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/53ead9c628aa4d9891fba5ea9279e49e 2024-11-20T05:15:04,414 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/B of d3eebe9eec8dc28c3c19437bf6136526 into 53ead9c628aa4d9891fba5ea9279e49e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:04,414 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:04,414 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/B, priority=13, startTime=1732079703959; duration=0sec 2024-11-20T05:15:04,414 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:04,414 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:B 2024-11-20T05:15:04,414 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:04,415 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36559 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:04,415 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/C is initiating minor compaction (all files) 2024-11-20T05:15:04,415 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/C in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:04,415 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/4d08f532f1124a9181881354131dd020, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/86f78494227d4f1982e1ae97a3b956aa, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/89e1b62d0185434f9e0fa2eb425c4c7c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=35.7 K 2024-11-20T05:15:04,416 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d08f532f1124a9181881354131dd020, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732079700971 2024-11-20T05:15:04,416 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f78494227d4f1982e1ae97a3b956aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732079701411 2024-11-20T05:15:04,417 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e1b62d0185434f9e0fa2eb425c4c7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732079702247 2024-11-20T05:15:04,417 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:04,418 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/6543f6167c8c4a3cbdc8057de4481a86 is 175, key is test_row_0/A:col10/1732079702269/Put/seqid=0 2024-11-20T05:15:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741974_1150 (size=31563) 2024-11-20T05:15:04,426 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:04,426 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/d0dc89a465584668886501fc3210c8e7 is 50, key is test_row_0/C:col10/1732079702269/Put/seqid=0 2024-11-20T05:15:04,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741975_1151 (size=12609) 2024-11-20T05:15:04,438 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/d0dc89a465584668886501fc3210c8e7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/d0dc89a465584668886501fc3210c8e7 2024-11-20T05:15:04,445 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/C of d3eebe9eec8dc28c3c19437bf6136526 into d0dc89a465584668886501fc3210c8e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:04,445 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:04,445 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/C, priority=13, startTime=1732079703959; duration=0sec 2024-11-20T05:15:04,445 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:04,445 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:C 2024-11-20T05:15:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:04,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:15:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:04,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dacdbdca812f4e52995a1d341ed4eecc_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079704468/Put/seqid=0 2024-11-20T05:15:04,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741976_1152 (size=12454) 2024-11-20T05:15:04,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079764493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:04,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079764494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079764597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079764597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T05:15:04,608 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:04,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-20T05:15:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T05:15:04,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:04,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:04,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T05:15:04,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:04,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079764807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:04,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079764807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:04,827 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/6543f6167c8c4a3cbdc8057de4481a86 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6543f6167c8c4a3cbdc8057de4481a86 2024-11-20T05:15:04,832 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/A of d3eebe9eec8dc28c3c19437bf6136526 into 6543f6167c8c4a3cbdc8057de4481a86(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:04,832 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:04,832 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/A, priority=13, startTime=1732079703959; duration=0sec 2024-11-20T05:15:04,832 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:04,832 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:A 2024-11-20T05:15:04,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:04,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dacdbdca812f4e52995a1d341ed4eecc_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dacdbdca812f4e52995a1d341ed4eecc_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:04,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/06e7efafdda7424fbd2d86696d665e0a, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:04,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/06e7efafdda7424fbd2d86696d665e0a is 175, key is test_row_0/A:col10/1732079704468/Put/seqid=0 2024-11-20T05:15:04,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741977_1153 (size=31255) 2024-11-20T05:15:04,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:04,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:04,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:04,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:04,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:04,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T05:15:05,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:05,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:05,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079765117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079765117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:05,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T05:15:05,296 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/06e7efafdda7424fbd2d86696d665e0a 2024-11-20T05:15:05,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/6d2415619c7448fe875f717b95937d63 is 50, key is test_row_0/B:col10/1732079704468/Put/seqid=0 2024-11-20T05:15:05,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741978_1154 (size=12301) 2024-11-20T05:15:05,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:05,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:05,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079765627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:05,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079765627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:05,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:05,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:05,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/6d2415619c7448fe875f717b95937d63 2024-11-20T05:15:05,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/1137c48efaf54b81a751ed3798924ec9 is 50, key is test_row_0/C:col10/1732079704468/Put/seqid=0 2024-11-20T05:15:05,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741979_1155 (size=12301) 2024-11-20T05:15:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T05:15:05,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:05,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:06,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/1137c48efaf54b81a751ed3798924ec9 2024-11-20T05:15:06,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/06e7efafdda7424fbd2d86696d665e0a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/06e7efafdda7424fbd2d86696d665e0a 2024-11-20T05:15:06,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:06,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:06,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/06e7efafdda7424fbd2d86696d665e0a, entries=150, sequenceid=304, filesize=30.5 K 2024-11-20T05:15:06,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/6d2415619c7448fe875f717b95937d63 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/6d2415619c7448fe875f717b95937d63 2024-11-20T05:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/6d2415619c7448fe875f717b95937d63, entries=150, sequenceid=304, filesize=12.0 K 2024-11-20T05:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/1137c48efaf54b81a751ed3798924ec9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/1137c48efaf54b81a751ed3798924ec9 2024-11-20T05:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/1137c48efaf54b81a751ed3798924ec9, entries=150, sequenceid=304, filesize=12.0 K 2024-11-20T05:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d3eebe9eec8dc28c3c19437bf6136526 in 1685ms, sequenceid=304, compaction requested=false 2024-11-20T05:15:06,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T05:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,291 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T05:15:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120328522c3964a4905aeeade8e04efed1b_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079704492/Put/seqid=0 2024-11-20T05:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741980_1156 (size=9914) 2024-11-20T05:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,320 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120328522c3964a4905aeeade8e04efed1b_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120328522c3964a4905aeeade8e04efed1b_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/9bce2b50e9584d8b8b6d4e0e58a719df, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:06,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/9bce2b50e9584d8b8b6d4e0e58a719df is 175, key is test_row_0/A:col10/1732079704492/Put/seqid=0 2024-11-20T05:15:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741981_1157 (size=22561) 2024-11-20T05:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:06,732 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/9bce2b50e9584d8b8b6d4e0e58a719df 2024-11-20T05:15:06,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/f9f909c9415246249b707a7f78c98772 is 50, key is test_row_0/B:col10/1732079704492/Put/seqid=0 2024-11-20T05:15:06,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079766738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079766740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T05:15:06,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741982_1158 (size=9857) 2024-11-20T05:15:06,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079766857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:06,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079766857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079767037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,041 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079767067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079767067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079767077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,078 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079767087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,092 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,163 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/f9f909c9415246249b707a7f78c98772 2024-11-20T05:15:07,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/70106888fc954b86ba48b8a4588a64af is 50, key is test_row_0/C:col10/1732079704492/Put/seqid=0 2024-11-20T05:15:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741983_1159 (size=9857) 2024-11-20T05:15:07,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079767378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079767378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,574 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/70106888fc954b86ba48b8a4588a64af 2024-11-20T05:15:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/9bce2b50e9584d8b8b6d4e0e58a719df as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/9bce2b50e9584d8b8b6d4e0e58a719df 2024-11-20T05:15:07,585 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/9bce2b50e9584d8b8b6d4e0e58a719df, entries=100, sequenceid=317, filesize=22.0 K 2024-11-20T05:15:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/f9f909c9415246249b707a7f78c98772 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f9f909c9415246249b707a7f78c98772 2024-11-20T05:15:07,591 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f9f909c9415246249b707a7f78c98772, entries=100, sequenceid=317, filesize=9.6 K 2024-11-20T05:15:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/70106888fc954b86ba48b8a4588a64af as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/70106888fc954b86ba48b8a4588a64af 2024-11-20T05:15:07,599 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/70106888fc954b86ba48b8a4588a64af, entries=100, sequenceid=317, filesize=9.6 K 2024-11-20T05:15:07,601 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d3eebe9eec8dc28c3c19437bf6136526 in 1309ms, sequenceid=317, compaction requested=true 2024-11-20T05:15:07,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:07,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:07,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-20T05:15:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-20T05:15:07,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-20T05:15:07,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9920 sec 2024-11-20T05:15:07,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.9960 sec 2024-11-20T05:15:07,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-20T05:15:07,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:07,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:07,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:07,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:07,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:07,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:07,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c01d132ad0ec41f2ba8b1ea24881360d_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079707888/Put/seqid=0 2024-11-20T05:15:07,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741984_1160 (size=17534) 2024-11-20T05:15:07,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079767909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079767910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079768017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079768017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079768227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079768227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:08,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:08,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c01d132ad0ec41f2ba8b1ea24881360d_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c01d132ad0ec41f2ba8b1ea24881360d_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:08,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/73913ccf96164f68bb610af987115f78, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:08,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/73913ccf96164f68bb610af987115f78 is 175, key is test_row_0/A:col10/1732079707888/Put/seqid=0 2024-11-20T05:15:08,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741985_1161 (size=48639) 2024-11-20T05:15:08,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079768537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:08,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079768537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:08,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/73913ccf96164f68bb610af987115f78 2024-11-20T05:15:08,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/045f781235364325a3953122d97db693 is 50, key is test_row_0/B:col10/1732079707888/Put/seqid=0 2024-11-20T05:15:08,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741986_1162 (size=12301) 2024-11-20T05:15:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T05:15:08,768 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:08,768 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T05:15:08,771 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T05:15:08,772 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:08,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T05:15:08,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T05:15:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:08,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:09,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079769047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:09,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079769047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:09,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T05:15:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T05:15:09,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/045f781235364325a3953122d97db693 2024-11-20T05:15:09,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/a20b2b9ff5fe4d009cffca335a2b50a7 is 50, key is test_row_0/C:col10/1732079707888/Put/seqid=0 2024-11-20T05:15:09,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741987_1163 (size=12301) 2024-11-20T05:15:09,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/a20b2b9ff5fe4d009cffca335a2b50a7 2024-11-20T05:15:09,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/73913ccf96164f68bb610af987115f78 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/73913ccf96164f68bb610af987115f78 2024-11-20T05:15:09,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/73913ccf96164f68bb610af987115f78, entries=250, sequenceid=339, filesize=47.5 K 2024-11-20T05:15:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/045f781235364325a3953122d97db693 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/045f781235364325a3953122d97db693 2024-11-20T05:15:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/045f781235364325a3953122d97db693, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T05:15:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/a20b2b9ff5fe4d009cffca335a2b50a7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a20b2b9ff5fe4d009cffca335a2b50a7 2024-11-20T05:15:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a20b2b9ff5fe4d009cffca335a2b50a7, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T05:15:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for d3eebe9eec8dc28c3c19437bf6136526 in 1323ms, sequenceid=339, compaction requested=true 2024-11-20T05:15:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:09,213 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:09,213 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,215 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47068 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:09,215 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/B is initiating minor compaction (all files) 2024-11-20T05:15:09,215 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/B in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,215 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/53ead9c628aa4d9891fba5ea9279e49e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/6d2415619c7448fe875f717b95937d63, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f9f909c9415246249b707a7f78c98772, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/045f781235364325a3953122d97db693] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=46.0 K 2024-11-20T05:15:09,215 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:09,215 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/A is initiating minor compaction (all files) 2024-11-20T05:15:09,216 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/A in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,216 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6543f6167c8c4a3cbdc8057de4481a86, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/06e7efafdda7424fbd2d86696d665e0a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/9bce2b50e9584d8b8b6d4e0e58a719df, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/73913ccf96164f68bb610af987115f78] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=130.9 K 2024-11-20T05:15:09,216 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,216 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6543f6167c8c4a3cbdc8057de4481a86, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/06e7efafdda7424fbd2d86696d665e0a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/9bce2b50e9584d8b8b6d4e0e58a719df, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/73913ccf96164f68bb610af987115f78] 2024-11-20T05:15:09,216 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ead9c628aa4d9891fba5ea9279e49e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732079702247 2024-11-20T05:15:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,216 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6543f6167c8c4a3cbdc8057de4481a86, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732079702247 2024-11-20T05:15:09,217 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d2415619c7448fe875f717b95937d63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732079702316 2024-11-20T05:15:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,219 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f909c9415246249b707a7f78c98772, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732079704487 2024-11-20T05:15:09,219 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e7efafdda7424fbd2d86696d665e0a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732079702316 2024-11-20T05:15:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,219 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bce2b50e9584d8b8b6d4e0e58a719df, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732079704487 2024-11-20T05:15:09,219 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 045f781235364325a3953122d97db693, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732079706731 2024-11-20T05:15:09,220 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73913ccf96164f68bb610af987115f78, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732079706729 2024-11-20T05:15:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T05:15:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,235 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:15:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:09,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:09,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,237 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,269 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#B#compaction#136 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,269 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/e3b55652a00e42e5981d387ab2e7c06f is 50, key is test_row_0/B:col10/1732079707888/Put/seqid=0 2024-11-20T05:15:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,273 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ed4926b12b324818a326704cc191fa43_d3eebe9eec8dc28c3c19437bf6136526 store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201a3d83abde4242cba37050900f9acabd_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_1/A:col10/1732079707907/Put/seqid=0 2024-11-20T05:15:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,285 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ed4926b12b324818a326704cc191fa43_d3eebe9eec8dc28c3c19437bf6136526, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:09,285 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ed4926b12b324818a326704cc191fa43_d3eebe9eec8dc28c3c19437bf6136526 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741988_1164 (size=12745) 2024-11-20T05:15:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741990_1166 (size=4469) 2024-11-20T05:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741989_1165 (size=9914) 2024-11-20T05:15:09,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,384 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201a3d83abde4242cba37050900f9acabd_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201a3d83abde4242cba37050900f9acabd_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:09,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/fcefb12c13a84255a12cd32a8ac7eb34, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/fcefb12c13a84255a12cd32a8ac7eb34 is 175, key is test_row_1/A:col10/1732079707907/Put/seqid=0 2024-11-20T05:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T05:15:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741991_1167 (size=22561) 2024-11-20T05:15:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,411 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/fcefb12c13a84255a12cd32a8ac7eb34 2024-11-20T05:15:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/98b3248838c14b1dbc2443ccd857a18a is 50, key is test_row_1/B:col10/1732079707907/Put/seqid=0 2024-11-20T05:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741992_1168 (size=9857) 2024-11-20T05:15:09,444 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/98b3248838c14b1dbc2443ccd857a18a 2024-11-20T05:15:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/62714f85401745509933e01d8b600228 is 50, key is test_row_1/C:col10/1732079707907/Put/seqid=0 2024-11-20T05:15:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741993_1169 (size=9857) 2024-11-20T05:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,492 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/62714f85401745509933e01d8b600228 2024-11-20T05:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/fcefb12c13a84255a12cd32a8ac7eb34 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/fcefb12c13a84255a12cd32a8ac7eb34 2024-11-20T05:15:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,503 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/fcefb12c13a84255a12cd32a8ac7eb34, entries=100, sequenceid=349, filesize=22.0 K 2024-11-20T05:15:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/98b3248838c14b1dbc2443ccd857a18a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/98b3248838c14b1dbc2443ccd857a18a 2024-11-20T05:15:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,519 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/98b3248838c14b1dbc2443ccd857a18a, entries=100, sequenceid=349, filesize=9.6 K 2024-11-20T05:15:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/62714f85401745509933e01d8b600228 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/62714f85401745509933e01d8b600228 2024-11-20T05:15:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,529 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/62714f85401745509933e01d8b600228, entries=100, sequenceid=349, filesize=9.6 K 2024-11-20T05:15:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,530 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for d3eebe9eec8dc28c3c19437bf6136526 in 295ms, sequenceid=349, compaction requested=true 2024-11-20T05:15:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T05:15:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T05:15:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T05:15:09,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 761 msec 2024-11-20T05:15:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 769 msec 2024-11-20T05:15:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,729 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/e3b55652a00e42e5981d387ab2e7c06f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e3b55652a00e42e5981d387ab2e7c06f 2024-11-20T05:15:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,734 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#A#compaction#135 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,734 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/165ff1127303478785d0f6b966795933 is 175, key is test_row_0/A:col10/1732079707888/Put/seqid=0 2024-11-20T05:15:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,736 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/B of d3eebe9eec8dc28c3c19437bf6136526 into e3b55652a00e42e5981d387ab2e7c06f(size=12.4 K), total size for store is 22.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,736 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:09,736 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/B, priority=12, startTime=1732079709213; duration=0sec 2024-11-20T05:15:09,736 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:09,737 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:B 2024-11-20T05:15:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,737 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T05:15:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,738 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 56925 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T05:15:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,738 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/C is initiating minor compaction (all files) 2024-11-20T05:15:09,739 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/C in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:09,739 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/d0dc89a465584668886501fc3210c8e7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/1137c48efaf54b81a751ed3798924ec9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/70106888fc954b86ba48b8a4588a64af, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a20b2b9ff5fe4d009cffca335a2b50a7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/62714f85401745509933e01d8b600228] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=55.6 K 2024-11-20T05:15:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,739 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d0dc89a465584668886501fc3210c8e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732079702247 2024-11-20T05:15:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,740 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1137c48efaf54b81a751ed3798924ec9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732079702316 2024-11-20T05:15:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,740 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 70106888fc954b86ba48b8a4588a64af, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732079704487 2024-11-20T05:15:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,740 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a20b2b9ff5fe4d009cffca335a2b50a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732079706731 2024-11-20T05:15:09,741 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 62714f85401745509933e01d8b600228, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732079707907 2024-11-20T05:15:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,780 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#C#compaction#140 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,780 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/bd274105343d49a8b650dd269fbe9452 is 50, key is test_row_0/C:col10/1732079707888/Put/seqid=0 2024-11-20T05:15:09,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741994_1170 (size=31699) 2024-11-20T05:15:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741995_1171 (size=12779) 2024-11-20T05:15:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T05:15:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,908 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T05:15:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T05:15:09,911 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,912 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:09,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T05:15:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T05:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T05:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T05:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T05:15:10,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-20T05:15:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 164 msec 2024-11-20T05:15:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,188 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/165ff1127303478785d0f6b966795933 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/165ff1127303478785d0f6b966795933 2024-11-20T05:15:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,195 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/A of d3eebe9eec8dc28c3c19437bf6136526 into 165ff1127303478785d0f6b966795933(size=31.0 K), total size for store is 53.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,195 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:10,195 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/A, priority=12, startTime=1732079709213; duration=0sec 2024-11-20T05:15:10,195 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:10,195 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:A 2024-11-20T05:15:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T05:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,227 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,228 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T05:15:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T05:15:10,230 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,231 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:10,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,251 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/bd274105343d49a8b650dd269fbe9452 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bd274105343d49a8b650dd269fbe9452 2024-11-20T05:15:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,257 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/C of d3eebe9eec8dc28c3c19437bf6136526 into bd274105343d49a8b650dd269fbe9452(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:10,257 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:10,257 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/C, priority=11, startTime=1732079709213; duration=0sec 2024-11-20T05:15:10,257 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:10,257 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:C 2024-11-20T05:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:15:10,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:10,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:10,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:10,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:10,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200c644efee4354274accabbac5fa60208_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079710272/Put/seqid=0 2024-11-20T05:15:10,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741996_1172 (size=22618) 2024-11-20T05:15:10,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:10,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200c644efee4354274accabbac5fa60208_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200c644efee4354274accabbac5fa60208_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:10,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5f11dafd22fc4f6bb3e965fc8568715f, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:10,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5f11dafd22fc4f6bb3e965fc8568715f is 175, key is test_row_0/A:col10/1732079710272/Put/seqid=0 2024-11-20T05:15:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T05:15:10,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741997_1173 (size=66023) 2024-11-20T05:15:10,383 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5f11dafd22fc4f6bb3e965fc8568715f 2024-11-20T05:15:10,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079710350, max timestamp=1732079710386Num uniques 859; 2024-11-20T05:15:10,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:10,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079710350, max timestamp=1732079710386Num uniques 859; 2024-11-20T05:15:10,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:10,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T05:15:10,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:10,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079710350, max timestamp=1732079710386Num uniques 859; 2024-11-20T05:15:10,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/f81e51dc95db4fe1a75d3a52b4eea3f2 is 50, key is test_row_0/B:col10/1732079710272/Put/seqid=0 2024-11-20T05:15:10,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741998_1174 (size=12301) 2024-11-20T05:15:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079770465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079770468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:10,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T05:15:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T05:15:10,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:10,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079770588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:10,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079770588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:10,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T05:15:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079770798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:10,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079770798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:10,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/f81e51dc95db4fe1a75d3a52b4eea3f2 2024-11-20T05:15:10,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/12085962dde04da8b7d2c8c78c1b8d6d is 50, key is test_row_0/C:col10/1732079710272/Put/seqid=0 2024-11-20T05:15:10,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T05:15:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:10,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T05:15:10,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741999_1175 (size=12301) 2024-11-20T05:15:11,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T05:15:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:11,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079771117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079771117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:11,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T05:15:11,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:11,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:11,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:11,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/12085962dde04da8b7d2c8c78c1b8d6d 2024-11-20T05:15:11,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5f11dafd22fc4f6bb3e965fc8568715f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5f11dafd22fc4f6bb3e965fc8568715f 2024-11-20T05:15:11,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5f11dafd22fc4f6bb3e965fc8568715f, entries=350, sequenceid=363, filesize=64.5 K 2024-11-20T05:15:11,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/f81e51dc95db4fe1a75d3a52b4eea3f2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f81e51dc95db4fe1a75d3a52b4eea3f2 2024-11-20T05:15:11,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f81e51dc95db4fe1a75d3a52b4eea3f2, entries=150, sequenceid=363, filesize=12.0 K 2024-11-20T05:15:11,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/12085962dde04da8b7d2c8c78c1b8d6d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/12085962dde04da8b7d2c8c78c1b8d6d 2024-11-20T05:15:11,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/12085962dde04da8b7d2c8c78c1b8d6d, entries=150, sequenceid=363, filesize=12.0 K 2024-11-20T05:15:11,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d3eebe9eec8dc28c3c19437bf6136526 in 1054ms, sequenceid=363, compaction requested=true 2024-11-20T05:15:11,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:11,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:11,327 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:11,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:11,328 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:11,329 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:11,329 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:11,329 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/A is initiating minor compaction (all files) 2024-11-20T05:15:11,329 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/B is initiating minor compaction (all files) 2024-11-20T05:15:11,329 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/A in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,329 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/B in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,329 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/165ff1127303478785d0f6b966795933, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/fcefb12c13a84255a12cd32a8ac7eb34, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5f11dafd22fc4f6bb3e965fc8568715f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=117.5 K 2024-11-20T05:15:11,329 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e3b55652a00e42e5981d387ab2e7c06f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/98b3248838c14b1dbc2443ccd857a18a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f81e51dc95db4fe1a75d3a52b4eea3f2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=34.1 K 2024-11-20T05:15:11,330 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,330 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/165ff1127303478785d0f6b966795933, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/fcefb12c13a84255a12cd32a8ac7eb34, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5f11dafd22fc4f6bb3e965fc8568715f] 2024-11-20T05:15:11,330 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e3b55652a00e42e5981d387ab2e7c06f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732079706731 2024-11-20T05:15:11,330 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165ff1127303478785d0f6b966795933, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732079706731 2024-11-20T05:15:11,331 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b3248838c14b1dbc2443ccd857a18a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732079707907 2024-11-20T05:15:11,331 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcefb12c13a84255a12cd32a8ac7eb34, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732079707907 2024-11-20T05:15:11,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T05:15:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:11,332 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f81e51dc95db4fe1a75d3a52b4eea3f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732079710212 2024-11-20T05:15:11,332 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T05:15:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:11,332 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f11dafd22fc4f6bb3e965fc8568715f, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732079710075 2024-11-20T05:15:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:11,353 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:11,355 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:11,355 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/9d3a2c54a73149d2bed4c5f1a8cca19c is 50, key is test_row_0/B:col10/1732079710272/Put/seqid=0 2024-11-20T05:15:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c1349093444143efae02e75dfd741a7a_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079710466/Put/seqid=0 2024-11-20T05:15:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T05:15:11,369 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a41440c81ba6477db75e6137f69c113b_d3eebe9eec8dc28c3c19437bf6136526 store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:11,371 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a41440c81ba6477db75e6137f69c113b_d3eebe9eec8dc28c3c19437bf6136526, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:11,372 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a41440c81ba6477db75e6137f69c113b_d3eebe9eec8dc28c3c19437bf6136526 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:11,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742000_1176 (size=12847) 2024-11-20T05:15:11,400 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/9d3a2c54a73149d2bed4c5f1a8cca19c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/9d3a2c54a73149d2bed4c5f1a8cca19c 2024-11-20T05:15:11,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742001_1177 (size=12454) 2024-11-20T05:15:11,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:11,416 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/B of d3eebe9eec8dc28c3c19437bf6136526 into 9d3a2c54a73149d2bed4c5f1a8cca19c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:11,416 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:11,416 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/B, priority=13, startTime=1732079711327; duration=0sec 2024-11-20T05:15:11,417 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:11,417 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:B 2024-11-20T05:15:11,417 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T05:15:11,418 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c1349093444143efae02e75dfd741a7a_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c1349093444143efae02e75dfd741a7a_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:11,419 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T05:15:11,419 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T05:15:11,419 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. because compaction request was cancelled 2024-11-20T05:15:11,419 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:C 2024-11-20T05:15:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e3df56d884e84cb6a7184fbea90b0e45, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e3df56d884e84cb6a7184fbea90b0e45 is 175, key is test_row_0/A:col10/1732079710466/Put/seqid=0 2024-11-20T05:15:11,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742002_1178 (size=4469) 2024-11-20T05:15:11,426 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#A#compaction#145 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:11,427 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/086f382b7e8a4537bfc758b99d9b5527 is 175, key is test_row_0/A:col10/1732079710272/Put/seqid=0 2024-11-20T05:15:11,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742003_1179 (size=31255) 2024-11-20T05:15:11,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742004_1180 (size=31801) 2024-11-20T05:15:11,457 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/086f382b7e8a4537bfc758b99d9b5527 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/086f382b7e8a4537bfc758b99d9b5527 2024-11-20T05:15:11,465 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/A of d3eebe9eec8dc28c3c19437bf6136526 into 086f382b7e8a4537bfc758b99d9b5527(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:11,465 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:11,465 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/A, priority=13, startTime=1732079711327; duration=0sec 2024-11-20T05:15:11,466 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:11,466 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:A 2024-11-20T05:15:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:11,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079771653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:11,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079771662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:11,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079771767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079771768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:11,838 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e3df56d884e84cb6a7184fbea90b0e45 2024-11-20T05:15:11,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/3dd10dede2a044cf8f696619a6571394 is 50, key is test_row_0/B:col10/1732079710466/Put/seqid=0 2024-11-20T05:15:11,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742005_1181 (size=12301) 2024-11-20T05:15:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079771977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079771977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:12,262 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/3dd10dede2a044cf8f696619a6571394 2024-11-20T05:15:12,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/bf1fe2c98592422eae83968c7b67c45b is 50, key is test_row_0/C:col10/1732079710466/Put/seqid=0 2024-11-20T05:15:12,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742006_1182 (size=12301) 2024-11-20T05:15:12,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079772287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:12,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079772287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T05:15:12,676 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/bf1fe2c98592422eae83968c7b67c45b 2024-11-20T05:15:12,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/e3df56d884e84cb6a7184fbea90b0e45 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e3df56d884e84cb6a7184fbea90b0e45 2024-11-20T05:15:12,685 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e3df56d884e84cb6a7184fbea90b0e45, entries=150, sequenceid=399, filesize=30.5 K 2024-11-20T05:15:12,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/3dd10dede2a044cf8f696619a6571394 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3dd10dede2a044cf8f696619a6571394 2024-11-20T05:15:12,690 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3dd10dede2a044cf8f696619a6571394, entries=150, sequenceid=399, filesize=12.0 K 2024-11-20T05:15:12,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/bf1fe2c98592422eae83968c7b67c45b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bf1fe2c98592422eae83968c7b67c45b 2024-11-20T05:15:12,696 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bf1fe2c98592422eae83968c7b67c45b, entries=150, sequenceid=399, filesize=12.0 K 2024-11-20T05:15:12,698 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for d3eebe9eec8dc28c3c19437bf6136526 in 1366ms, sequenceid=399, compaction requested=true 2024-11-20T05:15:12,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:12,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:12,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T05:15:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T05:15:12,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T05:15:12,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-11-20T05:15:12,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.4740 sec 2024-11-20T05:15:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:12,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:15:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:12,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:12,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:12,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:12,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aa7ee8db6038473fa242c764f7527a36_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079712799/Put/seqid=0 2024-11-20T05:15:12,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742007_1183 (size=14994) 2024-11-20T05:15:12,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079712829, max timestamp=1732079712843Num uniques 859; 2024-11-20T05:15:12,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079712829, max timestamp=1732079712843Num uniques 859; 2024-11-20T05:15:12,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:12,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079712829, max timestamp=1732079712843Num uniques 859; 2024-11-20T05:15:12,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:12,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079772902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079772907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:13,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079773017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:13,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079773017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:13,219 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:13,223 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aa7ee8db6038473fa242c764f7527a36_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aa7ee8db6038473fa242c764f7527a36_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:13,224 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5687ebe9e51e4b929682cc31bbf37af8, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:13,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5687ebe9e51e4b929682cc31bbf37af8 is 175, key is test_row_0/A:col10/1732079712799/Put/seqid=0 2024-11-20T05:15:13,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079773227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:13,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079773227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:13,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742008_1184 (size=39949) 2024-11-20T05:15:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:13,230 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5687ebe9e51e4b929682cc31bbf37af8 2024-11-20T05:15:13,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/3457f409544c43c2821757d8ee904d42 is 50, key is test_row_0/B:col10/1732079712799/Put/seqid=0 2024-11-20T05:15:13,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742009_1185 (size=12301) 2024-11-20T05:15:13,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/3457f409544c43c2821757d8ee904d42 2024-11-20T05:15:13,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/60ce0bc5a31c4482af4b794ae429c59c is 50, key is test_row_0/C:col10/1732079712799/Put/seqid=0 2024-11-20T05:15:13,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742010_1186 (size=12301) 2024-11-20T05:15:13,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079773537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:13,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079773537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:13,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/60ce0bc5a31c4482af4b794ae429c59c 2024-11-20T05:15:13,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5687ebe9e51e4b929682cc31bbf37af8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5687ebe9e51e4b929682cc31bbf37af8 2024-11-20T05:15:13,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5687ebe9e51e4b929682cc31bbf37af8, entries=200, sequenceid=412, filesize=39.0 K 2024-11-20T05:15:13,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/3457f409544c43c2821757d8ee904d42 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3457f409544c43c2821757d8ee904d42 2024-11-20T05:15:13,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3457f409544c43c2821757d8ee904d42, entries=150, sequenceid=412, filesize=12.0 K 2024-11-20T05:15:13,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/60ce0bc5a31c4482af4b794ae429c59c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/60ce0bc5a31c4482af4b794ae429c59c 2024-11-20T05:15:13,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/60ce0bc5a31c4482af4b794ae429c59c, entries=150, sequenceid=412, filesize=12.0 K 2024-11-20T05:15:13,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d3eebe9eec8dc28c3c19437bf6136526 in 883ms, sequenceid=412, compaction requested=true 2024-11-20T05:15:13,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:13,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:13,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:13,683 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:13,684 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:13,684 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:13,684 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:13,685 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/A is initiating minor compaction (all files) 2024-11-20T05:15:13,685 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/B is initiating minor compaction (all files) 2024-11-20T05:15:13,685 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/A in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:13,685 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/B in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:13,685 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/9d3a2c54a73149d2bed4c5f1a8cca19c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3dd10dede2a044cf8f696619a6571394, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3457f409544c43c2821757d8ee904d42] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=36.6 K 2024-11-20T05:15:13,685 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/086f382b7e8a4537bfc758b99d9b5527, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e3df56d884e84cb6a7184fbea90b0e45, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5687ebe9e51e4b929682cc31bbf37af8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=100.6 K 2024-11-20T05:15:13,685 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:13,685 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/086f382b7e8a4537bfc758b99d9b5527, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e3df56d884e84cb6a7184fbea90b0e45, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5687ebe9e51e4b929682cc31bbf37af8] 2024-11-20T05:15:13,685 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3a2c54a73149d2bed4c5f1a8cca19c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732079710212 2024-11-20T05:15:13,685 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 086f382b7e8a4537bfc758b99d9b5527, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732079710212 2024-11-20T05:15:13,686 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dd10dede2a044cf8f696619a6571394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732079710460 2024-11-20T05:15:13,686 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3df56d884e84cb6a7184fbea90b0e45, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732079710460 2024-11-20T05:15:13,686 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5687ebe9e51e4b929682cc31bbf37af8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732079711650 2024-11-20T05:15:13,686 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3457f409544c43c2821757d8ee904d42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732079711650 2024-11-20T05:15:13,695 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:13,696 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/500c8bc6ae9d4fcfaea71f2c0c02929b is 50, key is test_row_0/B:col10/1732079712799/Put/seqid=0 2024-11-20T05:15:13,704 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:13,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742011_1187 (size=12949) 2024-11-20T05:15:13,709 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202c872998d1f64fea88b1c70c15de0238_d3eebe9eec8dc28c3c19437bf6136526 store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:13,711 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202c872998d1f64fea88b1c70c15de0238_d3eebe9eec8dc28c3c19437bf6136526, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:13,711 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202c872998d1f64fea88b1c70c15de0238_d3eebe9eec8dc28c3c19437bf6136526 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:13,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742012_1188 (size=4469) 2024-11-20T05:15:13,719 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#A#compaction#153 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:13,720 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/4bf6d38541d14fbb8febce9885172541 is 175, key is test_row_0/A:col10/1732079712799/Put/seqid=0 2024-11-20T05:15:13,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742013_1189 (size=31903) 2024-11-20T05:15:13,789 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T05:15:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:14,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-20T05:15:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:14,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203c3e7b81ad7a4c00a399c2ed766877d7_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079712906/Put/seqid=0 2024-11-20T05:15:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742014_1190 (size=14994) 2024-11-20T05:15:14,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079774080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079774088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,114 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/500c8bc6ae9d4fcfaea71f2c0c02929b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/500c8bc6ae9d4fcfaea71f2c0c02929b 2024-11-20T05:15:14,118 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/B of d3eebe9eec8dc28c3c19437bf6136526 into 500c8bc6ae9d4fcfaea71f2c0c02929b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:14,118 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:14,118 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/B, priority=13, startTime=1732079713683; duration=0sec 2024-11-20T05:15:14,119 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:14,119 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:B 2024-11-20T05:15:14,119 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:14,120 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:14,120 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/C is initiating minor compaction (all files) 2024-11-20T05:15:14,120 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/C in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:14,120 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bd274105343d49a8b650dd269fbe9452, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/12085962dde04da8b7d2c8c78c1b8d6d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bf1fe2c98592422eae83968c7b67c45b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/60ce0bc5a31c4482af4b794ae429c59c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=48.5 K 2024-11-20T05:15:14,120 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bd274105343d49a8b650dd269fbe9452, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732079707888 2024-11-20T05:15:14,121 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 12085962dde04da8b7d2c8c78c1b8d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732079710212 2024-11-20T05:15:14,121 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1fe2c98592422eae83968c7b67c45b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732079710460 2024-11-20T05:15:14,121 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ce0bc5a31c4482af4b794ae429c59c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732079711650 2024-11-20T05:15:14,128 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:14,128 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/3e5383a0762b4fd588d1a780149b86b0 is 50, key is test_row_0/C:col10/1732079712799/Put/seqid=0 2024-11-20T05:15:14,134 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/4bf6d38541d14fbb8febce9885172541 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/4bf6d38541d14fbb8febce9885172541 2024-11-20T05:15:14,138 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/A of d3eebe9eec8dc28c3c19437bf6136526 into 4bf6d38541d14fbb8febce9885172541(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:14,139 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:14,139 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/A, priority=13, startTime=1732079713683; duration=0sec 2024-11-20T05:15:14,139 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:14,139 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:A 2024-11-20T05:15:14,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742015_1191 (size=12915) 2024-11-20T05:15:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079774197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079774197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T05:15:14,387 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:14,388 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T05:15:14,389 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:14,390 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:14,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T05:15:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079774407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079774408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,472 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:14,476 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203c3e7b81ad7a4c00a399c2ed766877d7_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203c3e7b81ad7a4c00a399c2ed766877d7_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:14,477 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/dfb5ef50be8e4cbc9e89743c9e1f46d1, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:14,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/dfb5ef50be8e4cbc9e89743c9e1f46d1 is 175, key is test_row_0/A:col10/1732079712906/Put/seqid=0 2024-11-20T05:15:14,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742016_1192 (size=39949) 2024-11-20T05:15:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T05:15:14,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T05:15:14,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:14,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:14,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:14,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,547 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/3e5383a0762b4fd588d1a780149b86b0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/3e5383a0762b4fd588d1a780149b86b0 2024-11-20T05:15:14,552 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/C of d3eebe9eec8dc28c3c19437bf6136526 into 3e5383a0762b4fd588d1a780149b86b0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:14,552 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:14,552 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/C, priority=12, startTime=1732079713684; duration=0sec 2024-11-20T05:15:14,552 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:14,553 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:C 2024-11-20T05:15:14,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T05:15:14,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:14,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:14,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:14,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T05:15:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079774717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079774717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:14,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T05:15:14,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:14,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:14,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:14,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:14,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/dfb5ef50be8e4cbc9e89743c9e1f46d1 2024-11-20T05:15:14,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/204dedbc038c4283b6f16b010a972bcc is 50, key is test_row_0/B:col10/1732079712906/Put/seqid=0 2024-11-20T05:15:14,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742017_1193 (size=12301) 2024-11-20T05:15:15,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T05:15:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:15,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T05:15:15,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T05:15:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:15,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079775228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079775228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:15,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/204dedbc038c4283b6f16b010a972bcc 2024-11-20T05:15:15,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/f54d893abaf74c2cae74e76c6ee34e49 is 50, key is test_row_0/C:col10/1732079712906/Put/seqid=0 2024-11-20T05:15:15,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T05:15:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:15,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:15,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742018_1194 (size=12301) 2024-11-20T05:15:15,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/f54d893abaf74c2cae74e76c6ee34e49 2024-11-20T05:15:15,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/dfb5ef50be8e4cbc9e89743c9e1f46d1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/dfb5ef50be8e4cbc9e89743c9e1f46d1 2024-11-20T05:15:15,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/dfb5ef50be8e4cbc9e89743c9e1f46d1, entries=200, sequenceid=447, filesize=39.0 K 2024-11-20T05:15:15,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/204dedbc038c4283b6f16b010a972bcc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/204dedbc038c4283b6f16b010a972bcc 2024-11-20T05:15:15,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/204dedbc038c4283b6f16b010a972bcc, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T05:15:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/f54d893abaf74c2cae74e76c6ee34e49 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/f54d893abaf74c2cae74e76c6ee34e49 2024-11-20T05:15:15,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/f54d893abaf74c2cae74e76c6ee34e49, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T05:15:15,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d3eebe9eec8dc28c3c19437bf6136526 in 1277ms, sequenceid=447, compaction requested=false 2024-11-20T05:15:15,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:15,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T05:15:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:15,460 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:15:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bce7a066820246f9ab025137f153caae_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079714079/Put/seqid=0 2024-11-20T05:15:15,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742019_1195 (size=12454) 2024-11-20T05:15:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T05:15:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:15,886 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bce7a066820246f9ab025137f153caae_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bce7a066820246f9ab025137f153caae_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/855fab0f77264510a35aed616d6ad9b9, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/855fab0f77264510a35aed616d6ad9b9 is 175, key is test_row_0/A:col10/1732079714079/Put/seqid=0 2024-11-20T05:15:15,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742020_1196 (size=31255) 2024-11-20T05:15:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:16,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. as already flushing 2024-11-20T05:15:16,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079716263, max timestamp=1732079716275Num uniques 859; 2024-11-20T05:15:16,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:16,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079716263, max timestamp=1732079716275Num uniques 859; 2024-11-20T05:15:16,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:16,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079716263, max timestamp=1732079716275Num uniques 859; 2024-11-20T05:15:16,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:16,302 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=460, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/855fab0f77264510a35aed616d6ad9b9 2024-11-20T05:15:16,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/680803d95e54441aa1b1a7851e1b6020 is 50, key is test_row_0/B:col10/1732079714079/Put/seqid=0 2024-11-20T05:15:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079776303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079776304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:16,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742021_1197 (size=12301) 2024-11-20T05:15:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079776417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079776418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:16,446 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-20T05:15:16,446 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:16,446 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,446 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:16,447 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-20T05:15:16,447 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:16,447 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,447 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:16,452 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-20T05:15:16,452 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:16,452 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,453 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:16,459 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-20T05:15:16,460 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:16,460 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,460 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:16,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T05:15:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079776628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079776628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:16,728 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/680803d95e54441aa1b1a7851e1b6020 2024-11-20T05:15:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/7fa26c88293c45d1a154051369922861 is 50, key is test_row_0/C:col10/1732079714079/Put/seqid=0 2024-11-20T05:15:16,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742022_1198 (size=12301) 2024-11-20T05:15:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46688 deadline: 1732079776938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46716 deadline: 1732079776938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46700 deadline: 1732079777128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:17,129 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:17,143 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/7fa26c88293c45d1a154051369922861 2024-11-20T05:15:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46638 deadline: 1732079777147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1732079777147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:17,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/855fab0f77264510a35aed616d6ad9b9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/855fab0f77264510a35aed616d6ad9b9 2024-11-20T05:15:17,148 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:17,148 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in d3eebe9eec8dc28c3c19437bf6136526 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:17,154 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/855fab0f77264510a35aed616d6ad9b9, entries=150, sequenceid=460, filesize=30.5 K 2024-11-20T05:15:17,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/680803d95e54441aa1b1a7851e1b6020 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/680803d95e54441aa1b1a7851e1b6020 2024-11-20T05:15:17,160 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/680803d95e54441aa1b1a7851e1b6020, entries=150, sequenceid=460, filesize=12.0 K 2024-11-20T05:15:17,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/7fa26c88293c45d1a154051369922861 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7fa26c88293c45d1a154051369922861 2024-11-20T05:15:17,165 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7fa26c88293c45d1a154051369922861, entries=150, sequenceid=460, filesize=12.0 K 2024-11-20T05:15:17,166 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d3eebe9eec8dc28c3c19437bf6136526 in 1706ms, sequenceid=460, compaction requested=true 2024-11-20T05:15:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T05:15:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T05:15:17,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T05:15:17,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7770 sec 2024-11-20T05:15:17,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.7810 sec 2024-11-20T05:15:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:17,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-20T05:15:17,451 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-11-20T05:15:17,451 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-11-20T05:15:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:17,451 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:17,451 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:17,451 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:17,451 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:17,451 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:17,451 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:17,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:17,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:17,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d48cfa86b2be4c728db3af2a74dedff1_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079716303/Put/seqid=0 2024-11-20T05:15:17,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742023_1199 (size=12454) 2024-11-20T05:15:17,866 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:17,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d48cfa86b2be4c728db3af2a74dedff1_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d48cfa86b2be4c728db3af2a74dedff1_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:17,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/7a59376faa62463ea8fb5480b5d7c7bd, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:17,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/7a59376faa62463ea8fb5480b5d7c7bd is 175, key is test_row_0/A:col10/1732079716303/Put/seqid=0 2024-11-20T05:15:17,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742024_1200 (size=31255) 2024-11-20T05:15:18,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=497, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/7a59376faa62463ea8fb5480b5d7c7bd 2024-11-20T05:15:18,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/803db9a01ea84ca7bfc244d0c0b5b9b1 is 50, key is test_row_0/B:col10/1732079716303/Put/seqid=0 2024-11-20T05:15:18,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742025_1201 (size=12301) 2024-11-20T05:15:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T05:15:18,558 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:18,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/803db9a01ea84ca7bfc244d0c0b5b9b1 2024-11-20T05:15:18,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/e468d303f36a4f558d699255142b41d7 is 50, key is test_row_0/C:col10/1732079716303/Put/seqid=0 2024-11-20T05:15:18,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742026_1202 (size=12301) 2024-11-20T05:15:19,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/e468d303f36a4f558d699255142b41d7 2024-11-20T05:15:19,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/7a59376faa62463ea8fb5480b5d7c7bd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/7a59376faa62463ea8fb5480b5d7c7bd 2024-11-20T05:15:19,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/7a59376faa62463ea8fb5480b5d7c7bd, entries=150, sequenceid=497, filesize=30.5 K 2024-11-20T05:15:19,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/803db9a01ea84ca7bfc244d0c0b5b9b1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/803db9a01ea84ca7bfc244d0c0b5b9b1 2024-11-20T05:15:19,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/803db9a01ea84ca7bfc244d0c0b5b9b1, entries=150, sequenceid=497, filesize=12.0 K 2024-11-20T05:15:19,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/e468d303f36a4f558d699255142b41d7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/e468d303f36a4f558d699255142b41d7 2024-11-20T05:15:19,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/e468d303f36a4f558d699255142b41d7, entries=150, sequenceid=497, filesize=12.0 K 2024-11-20T05:15:19,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=0 B/0 for d3eebe9eec8dc28c3c19437bf6136526 in 1699ms, sequenceid=497, compaction requested=true 2024-11-20T05:15:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:19,149 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3eebe9eec8dc28c3c19437bf6136526:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:19,149 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:19,150 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:19,150 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:19,150 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/B is initiating minor compaction (all files) 2024-11-20T05:15:19,150 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/A is initiating minor compaction (all files) 2024-11-20T05:15:19,150 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/A in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:19,150 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/B in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:19,150 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/4bf6d38541d14fbb8febce9885172541, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/dfb5ef50be8e4cbc9e89743c9e1f46d1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/855fab0f77264510a35aed616d6ad9b9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/7a59376faa62463ea8fb5480b5d7c7bd] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=131.2 K 2024-11-20T05:15:19,150 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/500c8bc6ae9d4fcfaea71f2c0c02929b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/204dedbc038c4283b6f16b010a972bcc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/680803d95e54441aa1b1a7851e1b6020, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/803db9a01ea84ca7bfc244d0c0b5b9b1] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=48.7 K 2024-11-20T05:15:19,150 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:19,150 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/4bf6d38541d14fbb8febce9885172541, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/dfb5ef50be8e4cbc9e89743c9e1f46d1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/855fab0f77264510a35aed616d6ad9b9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/7a59376faa62463ea8fb5480b5d7c7bd] 2024-11-20T05:15:19,151 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 500c8bc6ae9d4fcfaea71f2c0c02929b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732079711650 2024-11-20T05:15:19,151 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf6d38541d14fbb8febce9885172541, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732079711650 2024-11-20T05:15:19,151 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 204dedbc038c4283b6f16b010a972bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732079712861 2024-11-20T05:15:19,151 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfb5ef50be8e4cbc9e89743c9e1f46d1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732079712861 2024-11-20T05:15:19,151 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 680803d95e54441aa1b1a7851e1b6020, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732079714076 2024-11-20T05:15:19,151 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 855fab0f77264510a35aed616d6ad9b9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732079714076 2024-11-20T05:15:19,152 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 803db9a01ea84ca7bfc244d0c0b5b9b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732079716293 2024-11-20T05:15:19,152 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a59376faa62463ea8fb5480b5d7c7bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732079716293 2024-11-20T05:15:19,160 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:19,161 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#B#compaction#164 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:19,162 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/d00551619cd642388779b15420ef8594 is 50, key is test_row_0/B:col10/1732079716303/Put/seqid=0 2024-11-20T05:15:19,165 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411205b788f5992804d8b8c3862905d9fd0bc_d3eebe9eec8dc28c3c19437bf6136526 store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:19,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742027_1203 (size=13085) 2024-11-20T05:15:19,189 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411205b788f5992804d8b8c3862905d9fd0bc_d3eebe9eec8dc28c3c19437bf6136526, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:19,189 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205b788f5992804d8b8c3862905d9fd0bc_d3eebe9eec8dc28c3c19437bf6136526 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:19,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742028_1204 (size=4469) 2024-11-20T05:15:19,575 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/d00551619cd642388779b15420ef8594 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/d00551619cd642388779b15420ef8594 2024-11-20T05:15:19,582 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/B of d3eebe9eec8dc28c3c19437bf6136526 into d00551619cd642388779b15420ef8594(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:19,582 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:19,582 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/B, priority=12, startTime=1732079719149; duration=0sec 2024-11-20T05:15:19,582 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:19,582 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:B 2024-11-20T05:15:19,582 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:19,584 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:19,584 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): d3eebe9eec8dc28c3c19437bf6136526/C is initiating minor compaction (all files) 2024-11-20T05:15:19,584 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3eebe9eec8dc28c3c19437bf6136526/C in TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:19,584 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/3e5383a0762b4fd588d1a780149b86b0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/f54d893abaf74c2cae74e76c6ee34e49, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7fa26c88293c45d1a154051369922861, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/e468d303f36a4f558d699255142b41d7] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp, totalSize=48.7 K 2024-11-20T05:15:19,585 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e5383a0762b4fd588d1a780149b86b0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732079711650 2024-11-20T05:15:19,585 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f54d893abaf74c2cae74e76c6ee34e49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732079712861 2024-11-20T05:15:19,586 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa26c88293c45d1a154051369922861, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732079714076 2024-11-20T05:15:19,586 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e468d303f36a4f558d699255142b41d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732079716293 2024-11-20T05:15:19,595 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:19,596 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5b258e153d9c434e88690f3c065ab58d is 175, key is test_row_0/A:col10/1732079716303/Put/seqid=0 2024-11-20T05:15:19,599 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3eebe9eec8dc28c3c19437bf6136526#C#compaction#166 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:19,600 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/7b42ca685afc488780f45aba7ea9912f is 50, key is test_row_0/C:col10/1732079716303/Put/seqid=0 2024-11-20T05:15:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742029_1205 (size=32039) 2024-11-20T05:15:19,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742030_1206 (size=13051) 2024-11-20T05:15:20,015 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/5b258e153d9c434e88690f3c065ab58d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5b258e153d9c434e88690f3c065ab58d 2024-11-20T05:15:20,018 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/7b42ca685afc488780f45aba7ea9912f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7b42ca685afc488780f45aba7ea9912f 2024-11-20T05:15:20,022 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/A of d3eebe9eec8dc28c3c19437bf6136526 into 5b258e153d9c434e88690f3c065ab58d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:20,022 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:20,022 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/A, priority=12, startTime=1732079719149; duration=0sec 2024-11-20T05:15:20,022 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:20,022 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:A 2024-11-20T05:15:20,024 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3eebe9eec8dc28c3c19437bf6136526/C of d3eebe9eec8dc28c3c19437bf6136526 into 7b42ca685afc488780f45aba7ea9912f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:20,024 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3eebe9eec8dc28c3c19437bf6136526: 2024-11-20T05:15:20,024 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., storeName=d3eebe9eec8dc28c3c19437bf6136526/C, priority=12, startTime=1732079719149; duration=0sec 2024-11-20T05:15:20,024 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:20,024 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3eebe9eec8dc28c3c19437bf6136526:C 2024-11-20T05:15:27,189 INFO [Thread-580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-580. 2024-11-20T05:15:27,189 DEBUG [Thread-580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:27,189 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,190 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:27,210 INFO [Thread-582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-582. 2024-11-20T05:15:27,210 DEBUG [Thread-582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:27,210 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,210 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:27,221 INFO [Thread-584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-584. 2024-11-20T05:15:27,221 DEBUG [Thread-584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:27,221 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,221 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 226 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2504 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2715 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1107 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3314 rows 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-20T05:15:27,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-20T05:15:27,222 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:15:27,222 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:15:27,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:27,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:27,224 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T05:15:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T05:15:27,231 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079727230"}]},"ts":"1732079727230"} 2024-11-20T05:15:27,233 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T05:15:27,233 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T05:15:27,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:15:27,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, UNASSIGN}] 2024-11-20T05:15:27,236 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, UNASSIGN 2024-11-20T05:15:27,237 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:27,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, UNASSIGN because future has completed 2024-11-20T05:15:27,239 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:15:27,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:15:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T05:15:27,393 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:27,393 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:15:27,393 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing d3eebe9eec8dc28c3c19437bf6136526, disabling compactions & flushes 2024-11-20T05:15:27,394 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:27,394 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:27,394 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. after waiting 0 ms 2024-11-20T05:15:27,394 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:27,394 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing d3eebe9eec8dc28c3c19437bf6136526 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-20T05:15:27,395 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=A 2024-11-20T05:15:27,395 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:27,395 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=B 2024-11-20T05:15:27,395 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:27,396 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3eebe9eec8dc28c3c19437bf6136526, store=C 2024-11-20T05:15:27,396 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:27,404 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ef159dcdaee346aaac4a3f9e2df4ade6_d3eebe9eec8dc28c3c19437bf6136526 is 50, key is test_row_0/A:col10/1732079727218/Put/seqid=0 2024-11-20T05:15:27,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742031_1207 (size=7374) 2024-11-20T05:15:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T05:15:27,811 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:27,821 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ef159dcdaee346aaac4a3f9e2df4ade6_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ef159dcdaee346aaac4a3f9e2df4ade6_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:27,823 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/6f597939bc024f51af8de849fb772c8b, store: [table=TestAcidGuarantees family=A region=d3eebe9eec8dc28c3c19437bf6136526] 2024-11-20T05:15:27,824 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/6f597939bc024f51af8de849fb772c8b is 175, key is test_row_0/A:col10/1732079727218/Put/seqid=0 2024-11-20T05:15:27,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742032_1208 (size=13865) 2024-11-20T05:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T05:15:28,232 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=506, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/6f597939bc024f51af8de849fb772c8b 2024-11-20T05:15:28,245 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/bbd908552b99473b91db26194fd042d1 is 50, key is test_row_0/B:col10/1732079727218/Put/seqid=0 2024-11-20T05:15:28,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742033_1209 (size=7415) 2024-11-20T05:15:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T05:15:28,650 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/bbd908552b99473b91db26194fd042d1 2024-11-20T05:15:28,658 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/ebfa861a7d074ac698bcffc0b5bac6cf is 50, key is test_row_0/C:col10/1732079727218/Put/seqid=0 2024-11-20T05:15:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742034_1210 (size=7415) 2024-11-20T05:15:29,063 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/ebfa861a7d074ac698bcffc0b5bac6cf 2024-11-20T05:15:29,074 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/A/6f597939bc024f51af8de849fb772c8b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6f597939bc024f51af8de849fb772c8b 2024-11-20T05:15:29,079 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6f597939bc024f51af8de849fb772c8b, entries=50, sequenceid=506, filesize=13.5 K 2024-11-20T05:15:29,080 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/B/bbd908552b99473b91db26194fd042d1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/bbd908552b99473b91db26194fd042d1 2024-11-20T05:15:29,085 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/bbd908552b99473b91db26194fd042d1, entries=50, sequenceid=506, filesize=7.2 K 2024-11-20T05:15:29,085 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/.tmp/C/ebfa861a7d074ac698bcffc0b5bac6cf as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/ebfa861a7d074ac698bcffc0b5bac6cf 2024-11-20T05:15:29,089 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/ebfa861a7d074ac698bcffc0b5bac6cf, entries=50, sequenceid=506, filesize=7.2 K 2024-11-20T05:15:29,090 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d3eebe9eec8dc28c3c19437bf6136526 in 1696ms, sequenceid=506, compaction requested=false 2024-11-20T05:15:29,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/226632a2623f4ca8bf0d5a0cec2f55f7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/084ffed184f041cca62fea8c7e76cefd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/3c6863b8e65c4f708d0e09895f71f526, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/1cbcc359335f4ef691d071dd63e93a76, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/a51e0414b8614f5ba54a06c32135346d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/f9a2fd2f99b84a91af919245c9f3b1bd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e0e1a8486b7c4500a2d56212227aad5a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c6ed69e43ac74148a030ff05ae3500bd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6543f6167c8c4a3cbdc8057de4481a86, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/06e7efafdda7424fbd2d86696d665e0a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/9bce2b50e9584d8b8b6d4e0e58a719df, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/73913ccf96164f68bb610af987115f78, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/165ff1127303478785d0f6b966795933, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/fcefb12c13a84255a12cd32a8ac7eb34, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5f11dafd22fc4f6bb3e965fc8568715f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/086f382b7e8a4537bfc758b99d9b5527, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e3df56d884e84cb6a7184fbea90b0e45, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5687ebe9e51e4b929682cc31bbf37af8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/4bf6d38541d14fbb8febce9885172541, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/dfb5ef50be8e4cbc9e89743c9e1f46d1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/855fab0f77264510a35aed616d6ad9b9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/7a59376faa62463ea8fb5480b5d7c7bd] to archive 2024-11-20T05:15:29,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:15:29,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/226632a2623f4ca8bf0d5a0cec2f55f7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/226632a2623f4ca8bf0d5a0cec2f55f7 2024-11-20T05:15:29,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/084ffed184f041cca62fea8c7e76cefd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/084ffed184f041cca62fea8c7e76cefd 2024-11-20T05:15:29,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c5c6bc95f0ad4dcfb1afc7d6216f86c5 2024-11-20T05:15:29,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/3c6863b8e65c4f708d0e09895f71f526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/3c6863b8e65c4f708d0e09895f71f526 2024-11-20T05:15:29,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/1cbcc359335f4ef691d071dd63e93a76 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/1cbcc359335f4ef691d071dd63e93a76 2024-11-20T05:15:29,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/a51e0414b8614f5ba54a06c32135346d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/a51e0414b8614f5ba54a06c32135346d 2024-11-20T05:15:29,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/f9a2fd2f99b84a91af919245c9f3b1bd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/f9a2fd2f99b84a91af919245c9f3b1bd 2024-11-20T05:15:29,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e0e1a8486b7c4500a2d56212227aad5a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e0e1a8486b7c4500a2d56212227aad5a 2024-11-20T05:15:29,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c6ed69e43ac74148a030ff05ae3500bd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/c6ed69e43ac74148a030ff05ae3500bd 2024-11-20T05:15:29,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6543f6167c8c4a3cbdc8057de4481a86 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6543f6167c8c4a3cbdc8057de4481a86 2024-11-20T05:15:29,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/06e7efafdda7424fbd2d86696d665e0a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/06e7efafdda7424fbd2d86696d665e0a 2024-11-20T05:15:29,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/9bce2b50e9584d8b8b6d4e0e58a719df to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/9bce2b50e9584d8b8b6d4e0e58a719df 2024-11-20T05:15:29,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/73913ccf96164f68bb610af987115f78 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/73913ccf96164f68bb610af987115f78 2024-11-20T05:15:29,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/165ff1127303478785d0f6b966795933 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/165ff1127303478785d0f6b966795933 2024-11-20T05:15:29,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/fcefb12c13a84255a12cd32a8ac7eb34 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/fcefb12c13a84255a12cd32a8ac7eb34 2024-11-20T05:15:29,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5f11dafd22fc4f6bb3e965fc8568715f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5f11dafd22fc4f6bb3e965fc8568715f 2024-11-20T05:15:29,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/086f382b7e8a4537bfc758b99d9b5527 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/086f382b7e8a4537bfc758b99d9b5527 2024-11-20T05:15:29,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e3df56d884e84cb6a7184fbea90b0e45 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/e3df56d884e84cb6a7184fbea90b0e45 2024-11-20T05:15:29,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5687ebe9e51e4b929682cc31bbf37af8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5687ebe9e51e4b929682cc31bbf37af8 2024-11-20T05:15:29,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/4bf6d38541d14fbb8febce9885172541 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/4bf6d38541d14fbb8febce9885172541 2024-11-20T05:15:29,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/dfb5ef50be8e4cbc9e89743c9e1f46d1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/dfb5ef50be8e4cbc9e89743c9e1f46d1 2024-11-20T05:15:29,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/855fab0f77264510a35aed616d6ad9b9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/855fab0f77264510a35aed616d6ad9b9 2024-11-20T05:15:29,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/7a59376faa62463ea8fb5480b5d7c7bd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/7a59376faa62463ea8fb5480b5d7c7bd 2024-11-20T05:15:29,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/256637ed4af043cd8b84c1fac618eb30, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/86aef5b96f624ebdbfd0a129f98c5542, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/8f5260ada68b4be7bc312477c4c64ce5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/b4321c1757ed45f6964581a5f50f73ff, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0faeef0fbb0b499f99537ec087f36501, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0f86f1b0b6d64b7e83b95ae8c12178f8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0e2e594681334bfe8d01df212558f1f3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e5e31580a26e43ada7e6b2f11425058f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/53ead9c628aa4d9891fba5ea9279e49e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/073284b09db347d7a43ac885dbe541bc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/6d2415619c7448fe875f717b95937d63, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f9f909c9415246249b707a7f78c98772, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e3b55652a00e42e5981d387ab2e7c06f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/045f781235364325a3953122d97db693, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/98b3248838c14b1dbc2443ccd857a18a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/9d3a2c54a73149d2bed4c5f1a8cca19c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f81e51dc95db4fe1a75d3a52b4eea3f2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3dd10dede2a044cf8f696619a6571394, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/500c8bc6ae9d4fcfaea71f2c0c02929b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3457f409544c43c2821757d8ee904d42, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/204dedbc038c4283b6f16b010a972bcc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/680803d95e54441aa1b1a7851e1b6020, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/803db9a01ea84ca7bfc244d0c0b5b9b1] to archive 2024-11-20T05:15:29,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:15:29,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/256637ed4af043cd8b84c1fac618eb30 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/256637ed4af043cd8b84c1fac618eb30 2024-11-20T05:15:29,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/86aef5b96f624ebdbfd0a129f98c5542 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/86aef5b96f624ebdbfd0a129f98c5542 2024-11-20T05:15:29,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/8f5260ada68b4be7bc312477c4c64ce5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/8f5260ada68b4be7bc312477c4c64ce5 2024-11-20T05:15:29,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/b4321c1757ed45f6964581a5f50f73ff to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/b4321c1757ed45f6964581a5f50f73ff 2024-11-20T05:15:29,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0faeef0fbb0b499f99537ec087f36501 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0faeef0fbb0b499f99537ec087f36501 2024-11-20T05:15:29,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0f86f1b0b6d64b7e83b95ae8c12178f8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0f86f1b0b6d64b7e83b95ae8c12178f8 2024-11-20T05:15:29,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0e2e594681334bfe8d01df212558f1f3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/0e2e594681334bfe8d01df212558f1f3 2024-11-20T05:15:29,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e5e31580a26e43ada7e6b2f11425058f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e5e31580a26e43ada7e6b2f11425058f 2024-11-20T05:15:29,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/53ead9c628aa4d9891fba5ea9279e49e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/53ead9c628aa4d9891fba5ea9279e49e 2024-11-20T05:15:29,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/073284b09db347d7a43ac885dbe541bc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/073284b09db347d7a43ac885dbe541bc 2024-11-20T05:15:29,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/6d2415619c7448fe875f717b95937d63 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/6d2415619c7448fe875f717b95937d63 2024-11-20T05:15:29,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f9f909c9415246249b707a7f78c98772 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f9f909c9415246249b707a7f78c98772 2024-11-20T05:15:29,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e3b55652a00e42e5981d387ab2e7c06f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/e3b55652a00e42e5981d387ab2e7c06f 2024-11-20T05:15:29,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/045f781235364325a3953122d97db693 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/045f781235364325a3953122d97db693 2024-11-20T05:15:29,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/98b3248838c14b1dbc2443ccd857a18a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/98b3248838c14b1dbc2443ccd857a18a 2024-11-20T05:15:29,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/9d3a2c54a73149d2bed4c5f1a8cca19c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/9d3a2c54a73149d2bed4c5f1a8cca19c 2024-11-20T05:15:29,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f81e51dc95db4fe1a75d3a52b4eea3f2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/f81e51dc95db4fe1a75d3a52b4eea3f2 2024-11-20T05:15:29,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3dd10dede2a044cf8f696619a6571394 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3dd10dede2a044cf8f696619a6571394 2024-11-20T05:15:29,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/500c8bc6ae9d4fcfaea71f2c0c02929b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/500c8bc6ae9d4fcfaea71f2c0c02929b 2024-11-20T05:15:29,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3457f409544c43c2821757d8ee904d42 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/3457f409544c43c2821757d8ee904d42 2024-11-20T05:15:29,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/204dedbc038c4283b6f16b010a972bcc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/204dedbc038c4283b6f16b010a972bcc 2024-11-20T05:15:29,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/680803d95e54441aa1b1a7851e1b6020 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/680803d95e54441aa1b1a7851e1b6020 2024-11-20T05:15:29,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/803db9a01ea84ca7bfc244d0c0b5b9b1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/803db9a01ea84ca7bfc244d0c0b5b9b1 2024-11-20T05:15:29,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/68318fa6820b43df823ebe8c5693cbf6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/8f8af35bffbd46b8bd5aadcb6242c354, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/5872ea8308c9439888235a6b583e6866, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a024a757f37d4fa78871b455dfba457e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/53ba64d696444020999e50e1406f840d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/4d08f532f1124a9181881354131dd020, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/cf088029e7bc4724b14f0af79371bebc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/86f78494227d4f1982e1ae97a3b956aa, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/d0dc89a465584668886501fc3210c8e7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/89e1b62d0185434f9e0fa2eb425c4c7c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/1137c48efaf54b81a751ed3798924ec9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/70106888fc954b86ba48b8a4588a64af, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a20b2b9ff5fe4d009cffca335a2b50a7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bd274105343d49a8b650dd269fbe9452, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/62714f85401745509933e01d8b600228, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/12085962dde04da8b7d2c8c78c1b8d6d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bf1fe2c98592422eae83968c7b67c45b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/3e5383a0762b4fd588d1a780149b86b0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/60ce0bc5a31c4482af4b794ae429c59c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/f54d893abaf74c2cae74e76c6ee34e49, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7fa26c88293c45d1a154051369922861, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/e468d303f36a4f558d699255142b41d7] to archive 2024-11-20T05:15:29,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:15:29,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/68318fa6820b43df823ebe8c5693cbf6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/68318fa6820b43df823ebe8c5693cbf6 2024-11-20T05:15:29,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/8f8af35bffbd46b8bd5aadcb6242c354 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/8f8af35bffbd46b8bd5aadcb6242c354 2024-11-20T05:15:29,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/5872ea8308c9439888235a6b583e6866 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/5872ea8308c9439888235a6b583e6866 2024-11-20T05:15:29,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a024a757f37d4fa78871b455dfba457e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a024a757f37d4fa78871b455dfba457e 2024-11-20T05:15:29,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/53ba64d696444020999e50e1406f840d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/53ba64d696444020999e50e1406f840d 2024-11-20T05:15:29,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/4d08f532f1124a9181881354131dd020 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/4d08f532f1124a9181881354131dd020 2024-11-20T05:15:29,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/cf088029e7bc4724b14f0af79371bebc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/cf088029e7bc4724b14f0af79371bebc 2024-11-20T05:15:29,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/86f78494227d4f1982e1ae97a3b956aa to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/86f78494227d4f1982e1ae97a3b956aa 2024-11-20T05:15:29,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/d0dc89a465584668886501fc3210c8e7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/d0dc89a465584668886501fc3210c8e7 2024-11-20T05:15:29,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/89e1b62d0185434f9e0fa2eb425c4c7c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/89e1b62d0185434f9e0fa2eb425c4c7c 2024-11-20T05:15:29,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/1137c48efaf54b81a751ed3798924ec9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/1137c48efaf54b81a751ed3798924ec9 2024-11-20T05:15:29,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/70106888fc954b86ba48b8a4588a64af to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/70106888fc954b86ba48b8a4588a64af 2024-11-20T05:15:29,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a20b2b9ff5fe4d009cffca335a2b50a7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/a20b2b9ff5fe4d009cffca335a2b50a7 2024-11-20T05:15:29,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bd274105343d49a8b650dd269fbe9452 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bd274105343d49a8b650dd269fbe9452 2024-11-20T05:15:29,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/62714f85401745509933e01d8b600228 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/62714f85401745509933e01d8b600228 2024-11-20T05:15:29,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/12085962dde04da8b7d2c8c78c1b8d6d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/12085962dde04da8b7d2c8c78c1b8d6d 2024-11-20T05:15:29,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bf1fe2c98592422eae83968c7b67c45b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/bf1fe2c98592422eae83968c7b67c45b 2024-11-20T05:15:29,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/3e5383a0762b4fd588d1a780149b86b0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/3e5383a0762b4fd588d1a780149b86b0 2024-11-20T05:15:29,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/60ce0bc5a31c4482af4b794ae429c59c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/60ce0bc5a31c4482af4b794ae429c59c 2024-11-20T05:15:29,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/f54d893abaf74c2cae74e76c6ee34e49 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/f54d893abaf74c2cae74e76c6ee34e49 2024-11-20T05:15:29,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7fa26c88293c45d1a154051369922861 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7fa26c88293c45d1a154051369922861 2024-11-20T05:15:29,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/e468d303f36a4f558d699255142b41d7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/e468d303f36a4f558d699255142b41d7 2024-11-20T05:15:29,162 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/recovered.edits/509.seqid, newMaxSeqId=509, maxSeqId=4 2024-11-20T05:15:29,162 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. 2024-11-20T05:15:29,162 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for d3eebe9eec8dc28c3c19437bf6136526: Waiting for close lock at 1732079727393Running coprocessor pre-close hooks at 1732079727393Disabling compacts and flushes for region at 1732079727393Disabling writes for close at 1732079727394 (+1 ms)Obtaining lock to block concurrent updates at 1732079727394Preparing flush snapshotting stores in d3eebe9eec8dc28c3c19437bf6136526 at 1732079727394Finished memstore snapshotting TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732079727396 (+2 ms)Flushing stores of TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526. at 1732079727397 (+1 ms)Flushing d3eebe9eec8dc28c3c19437bf6136526/A: creating writer at 1732079727397Flushing mob file d3eebe9eec8dc28c3c19437bf6136526/A: appending metadata at 1732079727404 (+7 ms)Flushing mob file d3eebe9eec8dc28c3c19437bf6136526/A: closing flushed file at 1732079727404Flushing d3eebe9eec8dc28c3c19437bf6136526/A: appending metadata at 1732079727823 (+419 ms)Flushing d3eebe9eec8dc28c3c19437bf6136526/A: closing flushed file at 1732079727823Flushing d3eebe9eec8dc28c3c19437bf6136526/B: creating writer at 1732079728241 (+418 ms)Flushing d3eebe9eec8dc28c3c19437bf6136526/B: appending metadata at 1732079728244 (+3 ms)Flushing d3eebe9eec8dc28c3c19437bf6136526/B: closing flushed file at 1732079728244Flushing d3eebe9eec8dc28c3c19437bf6136526/C: creating writer at 1732079728654 (+410 ms)Flushing d3eebe9eec8dc28c3c19437bf6136526/C: appending metadata at 1732079728657 (+3 ms)Flushing d3eebe9eec8dc28c3c19437bf6136526/C: closing flushed file at 1732079728657Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a79458c: reopening flushed file at 1732079729073 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ae795c5: reopening flushed file at 1732079729079 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cb7c1b7: reopening flushed file at 1732079729085 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d3eebe9eec8dc28c3c19437bf6136526 in 1696ms, sequenceid=506, compaction requested=false at 1732079729090 (+5 ms)Writing region close event to WAL at 1732079729159 (+69 ms)Running coprocessor post-close hooks at 1732079729162 (+3 ms)Closed at 1732079729162 2024-11-20T05:15:29,165 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,165 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=d3eebe9eec8dc28c3c19437bf6136526, regionState=CLOSED 2024-11-20T05:15:29,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:15:29,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T05:15:29,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure d3eebe9eec8dc28c3c19437bf6136526, server=51d5d2011a1c,44753,1732079655401 in 1.9280 sec 2024-11-20T05:15:29,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-20T05:15:29,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3eebe9eec8dc28c3c19437bf6136526, UNASSIGN in 1.9340 sec 2024-11-20T05:15:29,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T05:15:29,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9370 sec 2024-11-20T05:15:29,174 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079729174"}]},"ts":"1732079729174"} 2024-11-20T05:15:29,177 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T05:15:29,177 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T05:15:29,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9540 sec 2024-11-20T05:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T05:15:29,378 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:29,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T05:15:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:29,380 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T05:15:29,380 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:29,383 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,386 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/recovered.edits] 2024-11-20T05:15:29,389 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5b258e153d9c434e88690f3c065ab58d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/5b258e153d9c434e88690f3c065ab58d 2024-11-20T05:15:29,390 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6f597939bc024f51af8de849fb772c8b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/A/6f597939bc024f51af8de849fb772c8b 2024-11-20T05:15:29,392 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/bbd908552b99473b91db26194fd042d1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/bbd908552b99473b91db26194fd042d1 2024-11-20T05:15:29,393 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/d00551619cd642388779b15420ef8594 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/B/d00551619cd642388779b15420ef8594 2024-11-20T05:15:29,395 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7b42ca685afc488780f45aba7ea9912f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/7b42ca685afc488780f45aba7ea9912f 2024-11-20T05:15:29,397 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/ebfa861a7d074ac698bcffc0b5bac6cf to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/C/ebfa861a7d074ac698bcffc0b5bac6cf 2024-11-20T05:15:29,399 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/recovered.edits/509.seqid to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526/recovered.edits/509.seqid 2024-11-20T05:15:29,399 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,399 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T05:15:29,400 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T05:15:29,401 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T05:15:29,404 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200c644efee4354274accabbac5fa60208_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200c644efee4354274accabbac5fa60208_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,405 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201a3d83abde4242cba37050900f9acabd_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201a3d83abde4242cba37050900f9acabd_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,406 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027fd68f5dbca4eaba0d1b3dd22f7f771_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027fd68f5dbca4eaba0d1b3dd22f7f771_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,408 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120328522c3964a4905aeeade8e04efed1b_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120328522c3964a4905aeeade8e04efed1b_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,408 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203c3e7b81ad7a4c00a399c2ed766877d7_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203c3e7b81ad7a4c00a399c2ed766877d7_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,409 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207d111aec10214d239a30afe36c388942_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207d111aec10214d239a30afe36c388942_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,410 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112090b649e80108476d9ee7a3d31243c9d8_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112090b649e80108476d9ee7a3d31243c9d8_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,411 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a3ca763893b94a39aa7be089a54f7e6f_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a3ca763893b94a39aa7be089a54f7e6f_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,412 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a4e3a4597e074ca2b188a6899d404238_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a4e3a4597e074ca2b188a6899d404238_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,413 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6cb2e74c12c4c01ab8be6017efcc6a4_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6cb2e74c12c4c01ab8be6017efcc6a4_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,414 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aa7ee8db6038473fa242c764f7527a36_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aa7ee8db6038473fa242c764f7527a36_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,415 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba3129ed17a540829b14a98f22cd7de8_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba3129ed17a540829b14a98f22cd7de8_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,416 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bce7a066820246f9ab025137f153caae_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bce7a066820246f9ab025137f153caae_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,417 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c01d132ad0ec41f2ba8b1ea24881360d_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c01d132ad0ec41f2ba8b1ea24881360d_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,418 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c1349093444143efae02e75dfd741a7a_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c1349093444143efae02e75dfd741a7a_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,419 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d48cfa86b2be4c728db3af2a74dedff1_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d48cfa86b2be4c728db3af2a74dedff1_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,420 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dacdbdca812f4e52995a1d341ed4eecc_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dacdbdca812f4e52995a1d341ed4eecc_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,421 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ef159dcdaee346aaac4a3f9e2df4ade6_d3eebe9eec8dc28c3c19437bf6136526 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ef159dcdaee346aaac4a3f9e2df4ade6_d3eebe9eec8dc28c3c19437bf6136526 2024-11-20T05:15:29,421 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T05:15:29,423 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:29,426 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T05:15:29,427 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T05:15:29,429 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:29,429 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T05:15:29,429 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732079729429"}]},"ts":"9223372036854775807"} 2024-11-20T05:15:29,431 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T05:15:29,431 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d3eebe9eec8dc28c3c19437bf6136526, NAME => 'TestAcidGuarantees,,1732079693254.d3eebe9eec8dc28c3c19437bf6136526.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T05:15:29,431 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T05:15:29,431 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732079729431"}]},"ts":"9223372036854775807"} 2024-11-20T05:15:29,433 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T05:15:29,433 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:29,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-20T05:15:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T05:15:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T05:15:29,488 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:29,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_142706796_22 at /127.0.0.1:38376 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=458 (was 430) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4606 (was 4864) 2024-11-20T05:15:29,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=458, ProcessCount=11, AvailableMemoryMB=4606 2024-11-20T05:15:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:15:29,514 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:15:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:29,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T05:15:29,517 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:29,517 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-20T05:15:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-20T05:15:29,518 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T05:15:29,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742035_1211 (size=1061) 2024-11-20T05:15:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-20T05:15:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-20T05:15:29,929 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:15:29,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742036_1212 (size=53) 2024-11-20T05:15:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-20T05:15:30,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:15:30,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 63a460fa7bbfa3a642b55ff183f6f407, disabling compactions & flushes 2024-11-20T05:15:30,340 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. after waiting 0 ms 2024-11-20T05:15:30,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,340 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,341 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63a460fa7bbfa3a642b55ff183f6f407: Waiting for close lock at 1732079730340Disabling compacts and flushes for region at 1732079730340Disabling writes for close at 1732079730340Writing region close event to WAL at 1732079730340Closed at 1732079730340 2024-11-20T05:15:30,343 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T05:15:30,344 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732079730343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732079730343"}]},"ts":"1732079730343"} 2024-11-20T05:15:30,348 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T05:15:30,350 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T05:15:30,350 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079730350"}]},"ts":"1732079730350"} 2024-11-20T05:15:30,353 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T05:15:30,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, ASSIGN}] 2024-11-20T05:15:30,354 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, ASSIGN 2024-11-20T05:15:30,355 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, ASSIGN; state=OFFLINE, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=false 2024-11-20T05:15:30,506 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=63a460fa7bbfa3a642b55ff183f6f407, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:30,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, ASSIGN because future has completed 2024-11-20T05:15:30,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:15:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-20T05:15:30,670 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,671 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:15:30,671 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,671 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:15:30,672 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,672 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,674 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,676 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:30,677 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63a460fa7bbfa3a642b55ff183f6f407 columnFamilyName A 2024-11-20T05:15:30,677 DEBUG [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:30,677 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.HStore(327): Store=63a460fa7bbfa3a642b55ff183f6f407/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:30,678 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,680 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:30,680 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63a460fa7bbfa3a642b55ff183f6f407 columnFamilyName B 2024-11-20T05:15:30,680 DEBUG [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:30,681 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.HStore(327): Store=63a460fa7bbfa3a642b55ff183f6f407/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:30,681 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,683 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:30,683 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63a460fa7bbfa3a642b55ff183f6f407 columnFamilyName C 2024-11-20T05:15:30,683 DEBUG [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:30,684 INFO [StoreOpener-63a460fa7bbfa3a642b55ff183f6f407-1 {}] regionserver.HStore(327): Store=63a460fa7bbfa3a642b55ff183f6f407/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:30,684 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,684 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,684 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,685 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,685 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,686 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,686 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:15:30,687 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,688 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:15:30,689 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 63a460fa7bbfa3a642b55ff183f6f407; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61149486, jitterRate=-0.08880165219306946}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:15:30,689 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:30,689 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 63a460fa7bbfa3a642b55ff183f6f407: Running coprocessor pre-open hook at 1732079730672Writing region info on filesystem at 1732079730672Initializing all the Stores at 1732079730673 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079730673Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079730674 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079730674Cleaning up temporary data from old regions at 1732079730686 (+12 ms)Running coprocessor post-open hooks at 1732079730689 (+3 ms)Region opened successfully at 1732079730689 2024-11-20T05:15:30,690 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., pid=57, masterSystemTime=1732079730664 2024-11-20T05:15:30,691 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,691 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:30,692 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=63a460fa7bbfa3a642b55ff183f6f407, regionState=OPEN, openSeqNum=2, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:30,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:15:30,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-20T05:15:30,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 in 183 msec 2024-11-20T05:15:30,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-20T05:15:30,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, ASSIGN in 342 msec 2024-11-20T05:15:30,697 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T05:15:30,697 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079730697"}]},"ts":"1732079730697"} 2024-11-20T05:15:30,698 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T05:15:30,699 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T05:15:30,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-20T05:15:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-20T05:15:31,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:31,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533c1a83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82032d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,672 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cd2de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,673 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T05:15:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 51d5d2011a1c,37927,1732079654666 2024-11-20T05:15:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3fdca488 2024-11-20T05:15:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T05:15:31,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T05:15:31,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2dccd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f280364, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56755949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661204ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,682 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d02280e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b8e622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40655b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,686 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@795046ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,687 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30468314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d355d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,689 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf97d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,690 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bbfae6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf4401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c79e3c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1ee6dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52267517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,696 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502aaba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,698 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fb3e41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4b43fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a44b00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a98a406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa78cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,703 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bc44e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76989242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d06ee61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,706 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c787448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,707 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e94e8b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a31e52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:31,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:31,709 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4546dc0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:31,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:31,710 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0df34e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:31,711 DEBUG [Thread-1011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,711 DEBUG [Thread-1020 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,712 DEBUG [Thread-1013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,712 DEBUG [Thread-1017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,712 DEBUG [Thread-1022 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,712 DEBUG [Thread-1024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,714 DEBUG [Thread-1026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,718 DEBUG [Thread-1028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,742 DEBUG [Thread-1015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-20T05:15:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T05:15:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:15:31,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:31,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:31,745 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:31,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,747 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:31,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,752 DEBUG [Thread-1009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:31,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:15:31,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/f85d5f088e1b4da7af323983d348e979 is 50, key is test_row_0/A:col10/1732079731720/Put/seqid=0 2024-11-20T05:15:31,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732079731751, max timestamp=1732079731773Num uniques 859; 2024-11-20T05:15:31,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:31,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079731751, max timestamp=1732079731773Num uniques 859; 2024-11-20T05:15:31,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:31,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079731751, max timestamp=1732079731773Num uniques 859; 2024-11-20T05:15:31,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:31,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079791801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079791807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079791809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742037_1213 (size=12001) 2024-11-20T05:15:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079791808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079791809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T05:15:31,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T05:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:31,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:31,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079791917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079791927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079791928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079791928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:31,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079791928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T05:15:32,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T05:15:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079792127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079792137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079792137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079792137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079792137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T05:15:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/f85d5f088e1b4da7af323983d348e979 2024-11-20T05:15:32,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/3e9cac4149b14123bc7e1ff3aa4f7a18 is 50, key is test_row_0/B:col10/1732079731720/Put/seqid=0 2024-11-20T05:15:32,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742038_1214 (size=12001) 2024-11-20T05:15:32,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T05:15:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T05:15:32,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079792437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079792447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079792447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079792447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079792447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T05:15:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T05:15:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:32,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/3e9cac4149b14123bc7e1ff3aa4f7a18 2024-11-20T05:15:32,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/0bd6b00cb5d0439ea1414440b980057d is 50, key is test_row_0/C:col10/1732079731720/Put/seqid=0 2024-11-20T05:15:32,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742039_1215 (size=12001) 2024-11-20T05:15:32,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/0bd6b00cb5d0439ea1414440b980057d 2024-11-20T05:15:32,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/f85d5f088e1b4da7af323983d348e979 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f85d5f088e1b4da7af323983d348e979 2024-11-20T05:15:32,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f85d5f088e1b4da7af323983d348e979, entries=150, sequenceid=11, filesize=11.7 K 2024-11-20T05:15:32,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/3e9cac4149b14123bc7e1ff3aa4f7a18 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3e9cac4149b14123bc7e1ff3aa4f7a18 2024-11-20T05:15:32,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3e9cac4149b14123bc7e1ff3aa4f7a18, entries=150, sequenceid=11, filesize=11.7 K 2024-11-20T05:15:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/0bd6b00cb5d0439ea1414440b980057d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0bd6b00cb5d0439ea1414440b980057d 2024-11-20T05:15:32,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0bd6b00cb5d0439ea1414440b980057d, entries=150, sequenceid=11, filesize=11.7 K 2024-11-20T05:15:32,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 63a460fa7bbfa3a642b55ff183f6f407 in 984ms, sequenceid=11, compaction requested=false 2024-11-20T05:15:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T05:15:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:32,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T05:15:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:32,822 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-20T05:15:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/bc3951a11e974f829256f222b68dce2a is 50, key is test_row_0/A:col10/1732079731807/Put/seqid=0 2024-11-20T05:15:32,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742040_1216 (size=12001) 2024-11-20T05:15:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T05:15:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:32,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079792960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079792961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079792962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079792963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079792965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079793077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079793077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079793077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079793078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079793078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,231 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/bc3951a11e974f829256f222b68dce2a 2024-11-20T05:15:33,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/690ec6400afa4ff5afb83c27b28a7f8f is 50, key is test_row_0/B:col10/1732079731807/Put/seqid=0 2024-11-20T05:15:33,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742041_1217 (size=12001) 2024-11-20T05:15:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079793287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079793287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079793288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079793288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079793288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079793597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079793597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079793598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079793598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079793598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:33,648 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/690ec6400afa4ff5afb83c27b28a7f8f 2024-11-20T05:15:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/cc41df560234471493030f2cb643ca95 is 50, key is test_row_0/C:col10/1732079731807/Put/seqid=0 2024-11-20T05:15:33,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742042_1218 (size=12001) 2024-11-20T05:15:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T05:15:34,059 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/cc41df560234471493030f2cb643ca95 2024-11-20T05:15:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/bc3951a11e974f829256f222b68dce2a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/bc3951a11e974f829256f222b68dce2a 2024-11-20T05:15:34,071 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/bc3951a11e974f829256f222b68dce2a, entries=150, sequenceid=46, filesize=11.7 K 2024-11-20T05:15:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/690ec6400afa4ff5afb83c27b28a7f8f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690ec6400afa4ff5afb83c27b28a7f8f 2024-11-20T05:15:34,075 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690ec6400afa4ff5afb83c27b28a7f8f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-20T05:15:34,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/cc41df560234471493030f2cb643ca95 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/cc41df560234471493030f2cb643ca95 2024-11-20T05:15:34,079 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/cc41df560234471493030f2cb643ca95, entries=150, sequenceid=46, filesize=11.7 K 2024-11-20T05:15:34,080 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 63a460fa7bbfa3a642b55ff183f6f407 in 1259ms, sequenceid=46, compaction requested=false 2024-11-20T05:15:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-20T05:15:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-20T05:15:34,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T05:15:34,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-11-20T05:15:34,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.3430 sec 2024-11-20T05:15:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:34,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:15:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:34,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/21075a6f24e846f0bfbd75bd5d8fed0c is 50, key is test_row_0/A:col10/1732079734109/Put/seqid=0 2024-11-20T05:15:34,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742043_1219 (size=12001) 2024-11-20T05:15:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079794126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079794131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079794131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079794132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079794132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079794237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079794247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079794247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079794248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079794248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079794447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079794457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079794458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079794458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079794458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/21075a6f24e846f0bfbd75bd5d8fed0c 2024-11-20T05:15:34,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/e070c70e722c4c6d94832721a8a3bae1 is 50, key is test_row_0/B:col10/1732079734109/Put/seqid=0 2024-11-20T05:15:34,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742044_1220 (size=12001) 2024-11-20T05:15:34,689 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T05:15:34,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079794757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079794767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079794767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079794767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079794767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:34,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/e070c70e722c4c6d94832721a8a3bae1 2024-11-20T05:15:34,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/f8db95036a3b4b279da832b2d5905d93 is 50, key is test_row_0/C:col10/1732079734109/Put/seqid=0 2024-11-20T05:15:34,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742045_1221 (size=12001) 2024-11-20T05:15:35,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079795267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:35,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079795277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079795277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:35,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079795278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079795278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:35,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/f8db95036a3b4b279da832b2d5905d93 2024-11-20T05:15:35,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/21075a6f24e846f0bfbd75bd5d8fed0c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/21075a6f24e846f0bfbd75bd5d8fed0c 2024-11-20T05:15:35,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/21075a6f24e846f0bfbd75bd5d8fed0c, entries=150, sequenceid=58, filesize=11.7 K 2024-11-20T05:15:35,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/e070c70e722c4c6d94832721a8a3bae1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/e070c70e722c4c6d94832721a8a3bae1 2024-11-20T05:15:35,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/e070c70e722c4c6d94832721a8a3bae1, entries=150, sequenceid=58, filesize=11.7 K 2024-11-20T05:15:35,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/f8db95036a3b4b279da832b2d5905d93 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f8db95036a3b4b279da832b2d5905d93 2024-11-20T05:15:35,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f8db95036a3b4b279da832b2d5905d93, entries=150, sequenceid=58, filesize=11.7 K 2024-11-20T05:15:35,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 63a460fa7bbfa3a642b55ff183f6f407 in 1252ms, sequenceid=58, compaction requested=true 2024-11-20T05:15:35,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:35,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:35,362 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:35,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:35,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:35,363 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:35,363 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:35,363 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:35,363 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:35,363 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:35,363 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:35,364 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:35,364 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f85d5f088e1b4da7af323983d348e979, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/bc3951a11e974f829256f222b68dce2a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/21075a6f24e846f0bfbd75bd5d8fed0c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.2 K 2024-11-20T05:15:35,364 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3e9cac4149b14123bc7e1ff3aa4f7a18, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690ec6400afa4ff5afb83c27b28a7f8f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/e070c70e722c4c6d94832721a8a3bae1] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.2 K 2024-11-20T05:15:35,364 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85d5f088e1b4da7af323983d348e979, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732079731718 2024-11-20T05:15:35,364 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e9cac4149b14123bc7e1ff3aa4f7a18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732079731718 2024-11-20T05:15:35,364 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc3951a11e974f829256f222b68dce2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732079731796 2024-11-20T05:15:35,365 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21075a6f24e846f0bfbd75bd5d8fed0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732079732954 2024-11-20T05:15:35,365 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 690ec6400afa4ff5afb83c27b28a7f8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732079731796 2024-11-20T05:15:35,365 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e070c70e722c4c6d94832721a8a3bae1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732079732954 2024-11-20T05:15:35,372 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:35,373 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#179 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:35,374 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/5e37c4eeaf574b13903ff159a1bdc40c is 50, key is test_row_0/A:col10/1732079734109/Put/seqid=0 2024-11-20T05:15:35,374 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/a02ac55a86fc4dd9ae38775f34b85df4 is 50, key is test_row_0/B:col10/1732079734109/Put/seqid=0 2024-11-20T05:15:35,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742046_1222 (size=12104) 2024-11-20T05:15:35,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742047_1223 (size=12104) 2024-11-20T05:15:35,807 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/5e37c4eeaf574b13903ff159a1bdc40c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5e37c4eeaf574b13903ff159a1bdc40c 2024-11-20T05:15:35,812 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into 5e37c4eeaf574b13903ff159a1bdc40c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:35,812 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:35,812 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=13, startTime=1732079735362; duration=0sec 2024-11-20T05:15:35,812 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:35,812 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:35,812 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:35,813 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:35,813 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:35,813 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:35,813 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0bd6b00cb5d0439ea1414440b980057d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/cc41df560234471493030f2cb643ca95, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f8db95036a3b4b279da832b2d5905d93] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.2 K 2024-11-20T05:15:35,814 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd6b00cb5d0439ea1414440b980057d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732079731718 2024-11-20T05:15:35,815 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc41df560234471493030f2cb643ca95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732079731796 2024-11-20T05:15:35,815 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8db95036a3b4b279da832b2d5905d93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732079732954 2024-11-20T05:15:35,816 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/a02ac55a86fc4dd9ae38775f34b85df4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/a02ac55a86fc4dd9ae38775f34b85df4 2024-11-20T05:15:35,821 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into a02ac55a86fc4dd9ae38775f34b85df4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:35,821 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:35,821 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=13, startTime=1732079735362; duration=0sec 2024-11-20T05:15:35,821 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:35,821 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:35,823 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:35,823 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4f536a7ce2cf4fc1b7d8898d55c19e7f is 50, key is test_row_0/C:col10/1732079734109/Put/seqid=0 2024-11-20T05:15:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742048_1224 (size=12104) 2024-11-20T05:15:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T05:15:35,908 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:35,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-20T05:15:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T05:15:35,910 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:35,910 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:35,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T05:15:36,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T05:15:36,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:36,062 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:15:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:36,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/7b4f5fd5203c4f889712726879f44e28 is 50, key is test_row_0/A:col10/1732079734131/Put/seqid=0 2024-11-20T05:15:36,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742049_1225 (size=12001) 2024-11-20T05:15:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T05:15:36,238 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4f536a7ce2cf4fc1b7d8898d55c19e7f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4f536a7ce2cf4fc1b7d8898d55c19e7f 2024-11-20T05:15:36,243 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into 4f536a7ce2cf4fc1b7d8898d55c19e7f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:36,243 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:36,243 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=13, startTime=1732079735363; duration=0sec 2024-11-20T05:15:36,243 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:36,243 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:36,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:36,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079796295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079796296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079796297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079796297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079796298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,301 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in 63a460fa7bbfa3a642b55ff183f6f407 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079796407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079796407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079796407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079796407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,470 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/7b4f5fd5203c4f889712726879f44e28 2024-11-20T05:15:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/738c2b25237b4d1191a1f4c1f15ba56f is 50, key is test_row_0/B:col10/1732079734131/Put/seqid=0 2024-11-20T05:15:36,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742050_1226 (size=12001) 2024-11-20T05:15:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T05:15:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079796617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079796617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079796617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079796617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,882 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/738c2b25237b4d1191a1f4c1f15ba56f 2024-11-20T05:15:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/5edc40ada6d345cfab84faf6d29004ab is 50, key is test_row_0/C:col10/1732079734131/Put/seqid=0 2024-11-20T05:15:36,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742051_1227 (size=12001) 2024-11-20T05:15:36,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079796927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079796927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079796928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079796928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T05:15:37,294 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/5edc40ada6d345cfab84faf6d29004ab 2024-11-20T05:15:37,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/7b4f5fd5203c4f889712726879f44e28 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/7b4f5fd5203c4f889712726879f44e28 2024-11-20T05:15:37,355 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/7b4f5fd5203c4f889712726879f44e28, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T05:15:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/738c2b25237b4d1191a1f4c1f15ba56f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/738c2b25237b4d1191a1f4c1f15ba56f 2024-11-20T05:15:37,360 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/738c2b25237b4d1191a1f4c1f15ba56f, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T05:15:37,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/5edc40ada6d345cfab84faf6d29004ab as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5edc40ada6d345cfab84faf6d29004ab 2024-11-20T05:15:37,365 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5edc40ada6d345cfab84faf6d29004ab, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T05:15:37,366 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63a460fa7bbfa3a642b55ff183f6f407 in 1304ms, sequenceid=80, compaction requested=false 2024-11-20T05:15:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-20T05:15:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-20T05:15:37,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T05:15:37,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-20T05:15:37,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.4610 sec 2024-11-20T05:15:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:37,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T05:15:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:37,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/19b8da6210f840f3ab13d621d86bfbb3 is 50, key is test_row_0/A:col10/1732079736296/Put/seqid=0 2024-11-20T05:15:37,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742052_1228 (size=12001) 2024-11-20T05:15:37,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/19b8da6210f840f3ab13d621d86bfbb3 2024-11-20T05:15:37,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/3420aecf181b4c7ab84766566a7d63e8 is 50, key is test_row_0/B:col10/1732079736296/Put/seqid=0 2024-11-20T05:15:37,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742053_1229 (size=12001) 2024-11-20T05:15:37,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/3420aecf181b4c7ab84766566a7d63e8 2024-11-20T05:15:37,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/93616162a39e4affa15cab84c6174a3b is 50, key is test_row_0/C:col10/1732079736296/Put/seqid=0 2024-11-20T05:15:37,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079797464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079797466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079797466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079797473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742054_1230 (size=12001) 2024-11-20T05:15:37,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/93616162a39e4affa15cab84c6174a3b 2024-11-20T05:15:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/19b8da6210f840f3ab13d621d86bfbb3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/19b8da6210f840f3ab13d621d86bfbb3 2024-11-20T05:15:37,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/19b8da6210f840f3ab13d621d86bfbb3, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T05:15:37,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/3420aecf181b4c7ab84766566a7d63e8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3420aecf181b4c7ab84766566a7d63e8 2024-11-20T05:15:37,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3420aecf181b4c7ab84766566a7d63e8, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T05:15:37,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/93616162a39e4affa15cab84c6174a3b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/93616162a39e4affa15cab84c6174a3b 2024-11-20T05:15:37,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/93616162a39e4affa15cab84c6174a3b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T05:15:37,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 63a460fa7bbfa3a642b55ff183f6f407 in 67ms, sequenceid=94, compaction requested=true 2024-11-20T05:15:37,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:37,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:37,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:37,507 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:37,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:37,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:37,508 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:37,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:37,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:37,508 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:37,509 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:37,509 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:37,509 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/a02ac55a86fc4dd9ae38775f34b85df4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/738c2b25237b4d1191a1f4c1f15ba56f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3420aecf181b4c7ab84766566a7d63e8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.3 K 2024-11-20T05:15:37,509 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:37,509 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a02ac55a86fc4dd9ae38775f34b85df4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732079732954 2024-11-20T05:15:37,509 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:37,509 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:37,510 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5e37c4eeaf574b13903ff159a1bdc40c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/7b4f5fd5203c4f889712726879f44e28, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/19b8da6210f840f3ab13d621d86bfbb3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.3 K 2024-11-20T05:15:37,510 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 738c2b25237b4d1191a1f4c1f15ba56f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732079734129 2024-11-20T05:15:37,510 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e37c4eeaf574b13903ff159a1bdc40c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732079732954 2024-11-20T05:15:37,510 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3420aecf181b4c7ab84766566a7d63e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732079736296 2024-11-20T05:15:37,510 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b4f5fd5203c4f889712726879f44e28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732079734129 2024-11-20T05:15:37,511 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b8da6210f840f3ab13d621d86bfbb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732079736296 2024-11-20T05:15:37,523 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:37,523 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:37,524 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/8b177c806d4b42509eeed7b041f373e7 is 50, key is test_row_0/B:col10/1732079736296/Put/seqid=0 2024-11-20T05:15:37,525 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/b27b687432db476baf33fae1949f296a is 50, key is test_row_0/A:col10/1732079736296/Put/seqid=0 2024-11-20T05:15:37,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742056_1232 (size=12207) 2024-11-20T05:15:37,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742055_1231 (size=12207) 2024-11-20T05:15:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:37,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:15:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:37,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/d8c4ba22500840799c457d01dcae40cf is 50, key is test_row_0/A:col10/1732079737456/Put/seqid=0 2024-11-20T05:15:37,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742057_1233 (size=14341) 2024-11-20T05:15:37,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/d8c4ba22500840799c457d01dcae40cf 2024-11-20T05:15:37,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/690becbd4997472daec5744f0fe53154 is 50, key is test_row_0/B:col10/1732079737456/Put/seqid=0 2024-11-20T05:15:37,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742058_1234 (size=12001) 2024-11-20T05:15:37,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079797596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079797598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079797599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079797609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079797707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079797717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079797727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079797738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079797917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079797927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079797937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,951 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/b27b687432db476baf33fae1949f296a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b27b687432db476baf33fae1949f296a 2024-11-20T05:15:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079797947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:37,954 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/8b177c806d4b42509eeed7b041f373e7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/8b177c806d4b42509eeed7b041f373e7 2024-11-20T05:15:37,955 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into b27b687432db476baf33fae1949f296a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:37,955 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:37,956 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=13, startTime=1732079737507; duration=0sec 2024-11-20T05:15:37,956 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:37,956 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:37,956 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:37,957 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:37,957 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:37,957 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:37,957 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4f536a7ce2cf4fc1b7d8898d55c19e7f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5edc40ada6d345cfab84faf6d29004ab, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/93616162a39e4affa15cab84c6174a3b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.3 K 2024-11-20T05:15:37,957 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f536a7ce2cf4fc1b7d8898d55c19e7f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732079732954 2024-11-20T05:15:37,958 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5edc40ada6d345cfab84faf6d29004ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732079734129 2024-11-20T05:15:37,958 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93616162a39e4affa15cab84c6174a3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732079736296 2024-11-20T05:15:37,958 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into 8b177c806d4b42509eeed7b041f373e7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:37,958 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:37,958 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=13, startTime=1732079737507; duration=0sec 2024-11-20T05:15:37,958 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:37,958 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:37,964 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:37,965 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/6f5fafc6668c46adaf8de68ab2d335d7 is 50, key is test_row_0/C:col10/1732079736296/Put/seqid=0 2024-11-20T05:15:37,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742059_1235 (size=12207) 2024-11-20T05:15:38,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/690becbd4997472daec5744f0fe53154 2024-11-20T05:15:38,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4d27221eaeab430eb3d14a1c4267dac5 is 50, key is test_row_0/C:col10/1732079737456/Put/seqid=0 2024-11-20T05:15:38,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742060_1236 (size=12001) 2024-11-20T05:15:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T05:15:38,058 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:38,058 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-20T05:15:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T05:15:38,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:38,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:38,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T05:15:38,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T05:15:38,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:38,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079798227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079798237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079798247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079798257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079798307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,311 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in 63a460fa7bbfa3a642b55ff183f6f407 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T05:15:38,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:38,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,375 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/6f5fafc6668c46adaf8de68ab2d335d7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6f5fafc6668c46adaf8de68ab2d335d7 2024-11-20T05:15:38,380 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into 6f5fafc6668c46adaf8de68ab2d335d7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:38,380 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:38,380 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=13, startTime=1732079737508; duration=0sec 2024-11-20T05:15:38,380 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:38,380 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T05:15:38,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4d27221eaeab430eb3d14a1c4267dac5 2024-11-20T05:15:38,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/d8c4ba22500840799c457d01dcae40cf as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d8c4ba22500840799c457d01dcae40cf 2024-11-20T05:15:38,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d8c4ba22500840799c457d01dcae40cf, entries=200, sequenceid=114, filesize=14.0 K 2024-11-20T05:15:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/690becbd4997472daec5744f0fe53154 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690becbd4997472daec5744f0fe53154 2024-11-20T05:15:38,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690becbd4997472daec5744f0fe53154, entries=150, sequenceid=114, filesize=11.7 K 2024-11-20T05:15:38,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4d27221eaeab430eb3d14a1c4267dac5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4d27221eaeab430eb3d14a1c4267dac5 2024-11-20T05:15:38,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4d27221eaeab430eb3d14a1c4267dac5, entries=150, sequenceid=114, filesize=11.7 K 2024-11-20T05:15:38,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 63a460fa7bbfa3a642b55ff183f6f407 in 851ms, sequenceid=114, compaction requested=false 2024-11-20T05:15:38,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:38,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T05:15:38,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,522 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T05:15:38,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:38,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:38,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:38,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:38,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:38,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:38,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/3b14d97753cc4828bd227adee6492ad0 is 50, key is test_row_0/A:col10/1732079737589/Put/seqid=0 2024-11-20T05:15:38,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742061_1237 (size=12001) 2024-11-20T05:15:38,532 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/3b14d97753cc4828bd227adee6492ad0 2024-11-20T05:15:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/c451c7fc921a40c1bb51de346f5a0ccc is 50, key is test_row_0/B:col10/1732079737589/Put/seqid=0 2024-11-20T05:15:38,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742062_1238 (size=12001) 2024-11-20T05:15:38,564 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/c451c7fc921a40c1bb51de346f5a0ccc 2024-11-20T05:15:38,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a5dc02f1d2284916becb707609cdf3a6 is 50, key is test_row_0/C:col10/1732079737589/Put/seqid=0 2024-11-20T05:15:38,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742063_1239 (size=12001) 2024-11-20T05:15:38,599 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a5dc02f1d2284916becb707609cdf3a6 2024-11-20T05:15:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/3b14d97753cc4828bd227adee6492ad0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/3b14d97753cc4828bd227adee6492ad0 2024-11-20T05:15:38,612 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/3b14d97753cc4828bd227adee6492ad0, entries=150, sequenceid=130, filesize=11.7 K 2024-11-20T05:15:38,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/c451c7fc921a40c1bb51de346f5a0ccc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c451c7fc921a40c1bb51de346f5a0ccc 2024-11-20T05:15:38,618 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c451c7fc921a40c1bb51de346f5a0ccc, entries=150, sequenceid=130, filesize=11.7 K 2024-11-20T05:15:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a5dc02f1d2284916becb707609cdf3a6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5dc02f1d2284916becb707609cdf3a6 2024-11-20T05:15:38,626 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5dc02f1d2284916becb707609cdf3a6, entries=150, sequenceid=130, filesize=11.7 K 2024-11-20T05:15:38,629 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 63a460fa7bbfa3a642b55ff183f6f407 in 107ms, sequenceid=130, compaction requested=true 2024-11-20T05:15:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-20T05:15:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-20T05:15:38,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-20T05:15:38,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 571 msec 2024-11-20T05:15:38,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 576 msec 2024-11-20T05:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T05:15:38,698 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:38,698 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-20T05:15:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T05:15:38,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:38,701 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:38,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:38,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:15:38,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:38,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:38,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:38,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:38,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:38,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:38,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/79dad19435a74f56929634369809b157 is 50, key is test_row_0/A:col10/1732079738749/Put/seqid=0 2024-11-20T05:15:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742064_1240 (size=14541) 2024-11-20T05:15:38,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/79dad19435a74f56929634369809b157 2024-11-20T05:15:38,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/33a735afd8e6418fad1ed93d8c8981de is 50, key is test_row_0/B:col10/1732079738749/Put/seqid=0 2024-11-20T05:15:38,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742065_1241 (size=12151) 2024-11-20T05:15:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079738777, max timestamp=1732079738786Num uniques 859; 2024-11-20T05:15:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079738777, max timestamp=1732079738786Num uniques 859; 2024-11-20T05:15:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079738777, max timestamp=1732079738786Num uniques 859; 2024-11-20T05:15:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T05:15:38,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079798806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079798807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079798810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079798810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T05:15:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:38,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:38,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079798917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079798917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079798927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079798927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T05:15:39,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:39,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T05:15:39,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079799127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079799127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079799137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079799137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T05:15:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/33a735afd8e6418fad1ed93d8c8981de 2024-11-20T05:15:39,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/3a4672b9044e4ae19e628c4ef477a8e2 is 50, key is test_row_0/C:col10/1732079738749/Put/seqid=0 2024-11-20T05:15:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742066_1242 (size=12151) 2024-11-20T05:15:39,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T05:15:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T05:15:39,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079799437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079799437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079799447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079799447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T05:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:39,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/3a4672b9044e4ae19e628c4ef477a8e2 2024-11-20T05:15:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/79dad19435a74f56929634369809b157 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/79dad19435a74f56929634369809b157 2024-11-20T05:15:39,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/79dad19435a74f56929634369809b157, entries=200, sequenceid=140, filesize=14.2 K 2024-11-20T05:15:39,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/33a735afd8e6418fad1ed93d8c8981de as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/33a735afd8e6418fad1ed93d8c8981de 2024-11-20T05:15:39,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/33a735afd8e6418fad1ed93d8c8981de, entries=150, sequenceid=140, filesize=11.9 K 2024-11-20T05:15:39,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/3a4672b9044e4ae19e628c4ef477a8e2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/3a4672b9044e4ae19e628c4ef477a8e2 2024-11-20T05:15:39,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/3a4672b9044e4ae19e628c4ef477a8e2, entries=150, sequenceid=140, filesize=11.9 K 2024-11-20T05:15:39,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 63a460fa7bbfa3a642b55ff183f6f407 in 848ms, sequenceid=140, compaction requested=true 2024-11-20T05:15:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:39,598 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:39,598 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:39,599 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:39,599 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:39,599 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:39,599 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:39,599 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,599 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,599 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/8b177c806d4b42509eeed7b041f373e7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690becbd4997472daec5744f0fe53154, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c451c7fc921a40c1bb51de346f5a0ccc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/33a735afd8e6418fad1ed93d8c8981de] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=47.2 K 2024-11-20T05:15:39,599 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b27b687432db476baf33fae1949f296a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d8c4ba22500840799c457d01dcae40cf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/3b14d97753cc4828bd227adee6492ad0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/79dad19435a74f56929634369809b157] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=51.8 K 2024-11-20T05:15:39,600 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27b687432db476baf33fae1949f296a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732079736296 2024-11-20T05:15:39,600 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b177c806d4b42509eeed7b041f373e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732079736296 2024-11-20T05:15:39,600 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8c4ba22500840799c457d01dcae40cf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732079737456 2024-11-20T05:15:39,600 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 690becbd4997472daec5744f0fe53154, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732079737456 2024-11-20T05:15:39,600 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b14d97753cc4828bd227adee6492ad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732079737589 2024-11-20T05:15:39,600 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c451c7fc921a40c1bb51de346f5a0ccc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732079737589 2024-11-20T05:15:39,600 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79dad19435a74f56929634369809b157, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732079738744 2024-11-20T05:15:39,601 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a735afd8e6418fad1ed93d8c8981de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732079738744 2024-11-20T05:15:39,608 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:39,608 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/56f516c4ea9746aa97eb1aac07e59e3a is 50, key is test_row_0/A:col10/1732079738749/Put/seqid=0 2024-11-20T05:15:39,610 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:39,610 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/03dac48d24404cb6bbb1d4eeb1cd8263 is 50, key is test_row_0/B:col10/1732079738749/Put/seqid=0 2024-11-20T05:15:39,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T05:15:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,617 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-20T05:15:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:39,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/de2c9e97c42f4770935630d11e8900b2 is 50, key is test_row_0/A:col10/1732079738809/Put/seqid=0 2024-11-20T05:15:39,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742068_1244 (size=12493) 2024-11-20T05:15:39,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742067_1243 (size=12493) 2024-11-20T05:15:39,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742069_1245 (size=12151) 2024-11-20T05:15:39,643 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/de2c9e97c42f4770935630d11e8900b2 2024-11-20T05:15:39,646 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/03dac48d24404cb6bbb1d4eeb1cd8263 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/03dac48d24404cb6bbb1d4eeb1cd8263 2024-11-20T05:15:39,657 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into 03dac48d24404cb6bbb1d4eeb1cd8263(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:39,657 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:39,657 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=12, startTime=1732079739598; duration=0sec 2024-11-20T05:15:39,657 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:39,657 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:39,657 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:39,659 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:39,659 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:39,660 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:39,660 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6f5fafc6668c46adaf8de68ab2d335d7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4d27221eaeab430eb3d14a1c4267dac5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5dc02f1d2284916becb707609cdf3a6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/3a4672b9044e4ae19e628c4ef477a8e2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=47.2 K 2024-11-20T05:15:39,660 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5fafc6668c46adaf8de68ab2d335d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732079736296 2024-11-20T05:15:39,661 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d27221eaeab430eb3d14a1c4267dac5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732079737456 2024-11-20T05:15:39,661 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a5dc02f1d2284916becb707609cdf3a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732079737589 2024-11-20T05:15:39,663 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4672b9044e4ae19e628c4ef477a8e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732079738744 2024-11-20T05:15:39,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/277086ec64854d0099e94ba8a6c9cf10 is 50, key is test_row_0/B:col10/1732079738809/Put/seqid=0 2024-11-20T05:15:39,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742070_1246 (size=12151) 2024-11-20T05:15:39,690 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:39,691 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/027a8ee3f3204231ba5f935b23a6a9f8 is 50, key is test_row_0/C:col10/1732079738749/Put/seqid=0 2024-11-20T05:15:39,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742071_1247 (size=12493) 2024-11-20T05:15:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T05:15:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:39,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:39,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079799957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079799961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079799961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:39,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079799967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,045 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/56f516c4ea9746aa97eb1aac07e59e3a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/56f516c4ea9746aa97eb1aac07e59e3a 2024-11-20T05:15:40,049 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into 56f516c4ea9746aa97eb1aac07e59e3a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:40,049 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:40,049 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=12, startTime=1732079739598; duration=0sec 2024-11-20T05:15:40,049 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:40,049 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079800067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079800077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079800077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,087 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/277086ec64854d0099e94ba8a6c9cf10 2024-11-20T05:15:40,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4e3d8771338a4b69a84ac7ebea1521d7 is 50, key is test_row_0/C:col10/1732079738809/Put/seqid=0 2024-11-20T05:15:40,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742072_1248 (size=12151) 2024-11-20T05:15:40,103 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/027a8ee3f3204231ba5f935b23a6a9f8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/027a8ee3f3204231ba5f935b23a6a9f8 2024-11-20T05:15:40,107 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into 027a8ee3f3204231ba5f935b23a6a9f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:40,107 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:40,107 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=12, startTime=1732079739598; duration=0sec 2024-11-20T05:15:40,107 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:40,107 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079800278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079800287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079800287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,498 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4e3d8771338a4b69a84ac7ebea1521d7 2024-11-20T05:15:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/de2c9e97c42f4770935630d11e8900b2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/de2c9e97c42f4770935630d11e8900b2 2024-11-20T05:15:40,515 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/de2c9e97c42f4770935630d11e8900b2, entries=150, sequenceid=176, filesize=11.9 K 2024-11-20T05:15:40,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/277086ec64854d0099e94ba8a6c9cf10 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/277086ec64854d0099e94ba8a6c9cf10 2024-11-20T05:15:40,520 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/277086ec64854d0099e94ba8a6c9cf10, entries=150, sequenceid=176, filesize=11.9 K 2024-11-20T05:15:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4e3d8771338a4b69a84ac7ebea1521d7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4e3d8771338a4b69a84ac7ebea1521d7 2024-11-20T05:15:40,524 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4e3d8771338a4b69a84ac7ebea1521d7, entries=150, sequenceid=176, filesize=11.9 K 2024-11-20T05:15:40,525 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 63a460fa7bbfa3a642b55ff183f6f407 in 909ms, sequenceid=176, compaction requested=false 2024-11-20T05:15:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-20T05:15:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-20T05:15:40,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T05:15:40,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8260 sec 2024-11-20T05:15:40,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8310 sec 2024-11-20T05:15:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:40,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:15:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:40,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/ce6cf3f59f974d4ab58771980ef41494 is 50, key is test_row_0/A:col10/1732079739960/Put/seqid=0 2024-11-20T05:15:40,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742073_1249 (size=14541) 2024-11-20T05:15:40,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079740608, max timestamp=1732079740623Num uniques 859; 2024-11-20T05:15:40,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:40,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079740608, max timestamp=1732079740623Num uniques 859; 2024-11-20T05:15:40,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079740608, max timestamp=1732079740623Num uniques 859; 2024-11-20T05:15:40,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:40,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079800655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079800656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079800657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079800767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079800767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079800767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T05:15:40,857 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:40,858 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-20T05:15:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T05:15:40,859 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:40,860 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:40,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T05:15:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079800977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079800977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079800978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079800978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:40,982 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in 63a460fa7bbfa3a642b55ff183f6f407 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/ce6cf3f59f974d4ab58771980ef41494 2024-11-20T05:15:41,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b9fd459db27c46d185f6a5ab3bef1843 is 50, key is test_row_0/B:col10/1732079739960/Put/seqid=0 2024-11-20T05:15:41,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T05:15:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742074_1250 (size=12151) 2024-11-20T05:15:41,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T05:15:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:41,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T05:15:41,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079801287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079801287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079801287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T05:15:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b9fd459db27c46d185f6a5ab3bef1843 2024-11-20T05:15:41,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/641bffa503aa4ca6962dde908383863c is 50, key is test_row_0/C:col10/1732079739960/Put/seqid=0 2024-11-20T05:15:41,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742075_1251 (size=12151) 2024-11-20T05:15:41,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/641bffa503aa4ca6962dde908383863c 2024-11-20T05:15:41,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/ce6cf3f59f974d4ab58771980ef41494 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ce6cf3f59f974d4ab58771980ef41494 2024-11-20T05:15:41,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ce6cf3f59f974d4ab58771980ef41494, entries=200, sequenceid=189, filesize=14.2 K 2024-11-20T05:15:41,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T05:15:41,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:41,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b9fd459db27c46d185f6a5ab3bef1843 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b9fd459db27c46d185f6a5ab3bef1843 2024-11-20T05:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:41,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b9fd459db27c46d185f6a5ab3bef1843, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T05:15:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/641bffa503aa4ca6962dde908383863c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/641bffa503aa4ca6962dde908383863c 2024-11-20T05:15:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/641bffa503aa4ca6962dde908383863c, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T05:15:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 63a460fa7bbfa3a642b55ff183f6f407 in 895ms, sequenceid=189, compaction requested=true 2024-11-20T05:15:41,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:41,486 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:41,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:41,487 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:41,487 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:41,487 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,487 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/03dac48d24404cb6bbb1d4eeb1cd8263, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/277086ec64854d0099e94ba8a6c9cf10, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b9fd459db27c46d185f6a5ab3bef1843] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.9 K 2024-11-20T05:15:41,487 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:41,487 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:41,487 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,487 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/56f516c4ea9746aa97eb1aac07e59e3a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/de2c9e97c42f4770935630d11e8900b2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ce6cf3f59f974d4ab58771980ef41494] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=38.3 K 2024-11-20T05:15:41,487 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 03dac48d24404cb6bbb1d4eeb1cd8263, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732079738744 2024-11-20T05:15:41,488 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f516c4ea9746aa97eb1aac07e59e3a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732079738744 2024-11-20T05:15:41,488 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting de2c9e97c42f4770935630d11e8900b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732079738806 2024-11-20T05:15:41,488 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 277086ec64854d0099e94ba8a6c9cf10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732079738806 2024-11-20T05:15:41,489 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6cf3f59f974d4ab58771980ef41494, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732079739955 2024-11-20T05:15:41,489 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fd459db27c46d185f6a5ab3bef1843, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732079739960 2024-11-20T05:15:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T05:15:41,495 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:41,496 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:41,496 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/cc5175151ccc48c6b8988c2cb4a0b948 is 50, key is test_row_0/A:col10/1732079739960/Put/seqid=0 2024-11-20T05:15:41,496 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/5fee276283d749c9a4e021fc0abead78 is 50, key is test_row_0/B:col10/1732079739960/Put/seqid=0 2024-11-20T05:15:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742076_1252 (size=12595) 2024-11-20T05:15:41,522 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/5fee276283d749c9a4e021fc0abead78 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/5fee276283d749c9a4e021fc0abead78 2024-11-20T05:15:41,528 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into 5fee276283d749c9a4e021fc0abead78(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:41,528 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:41,529 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=13, startTime=1732079741486; duration=0sec 2024-11-20T05:15:41,529 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:41,529 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:41,529 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:41,530 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:41,530 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:41,530 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,531 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/027a8ee3f3204231ba5f935b23a6a9f8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4e3d8771338a4b69a84ac7ebea1521d7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/641bffa503aa4ca6962dde908383863c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=35.9 K 2024-11-20T05:15:41,532 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 027a8ee3f3204231ba5f935b23a6a9f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732079738744 2024-11-20T05:15:41,532 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e3d8771338a4b69a84ac7ebea1521d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732079738806 2024-11-20T05:15:41,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742077_1253 (size=12595) 2024-11-20T05:15:41,535 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 641bffa503aa4ca6962dde908383863c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732079739960 2024-11-20T05:15:41,541 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/cc5175151ccc48c6b8988c2cb4a0b948 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/cc5175151ccc48c6b8988c2cb4a0b948 2024-11-20T05:15:41,548 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into cc5175151ccc48c6b8988c2cb4a0b948(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:41,548 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:41,548 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=13, startTime=1732079741486; duration=0sec 2024-11-20T05:15:41,548 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:41,548 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:41,553 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:41,553 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4c82757075bf42d2a7f6e921fd193e99 is 50, key is test_row_0/C:col10/1732079739960/Put/seqid=0 2024-11-20T05:15:41,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742078_1254 (size=12595) 2024-11-20T05:15:41,571 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/4c82757075bf42d2a7f6e921fd193e99 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4c82757075bf42d2a7f6e921fd193e99 2024-11-20T05:15:41,576 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into 4c82757075bf42d2a7f6e921fd193e99(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:41,576 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:41,576 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=13, startTime=1732079741486; duration=0sec 2024-11-20T05:15:41,576 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:41,576 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:41,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T05:15:41,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:41,624 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-20T05:15:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:41,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/dd237485e03a4cc685b2e4e57024d787 is 50, key is test_row_0/A:col10/1732079740653/Put/seqid=0 2024-11-20T05:15:41,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742079_1255 (size=12151) 2024-11-20T05:15:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:41,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:41,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079801808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079801815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079801815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079801927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079801928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079801928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T05:15:42,066 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/dd237485e03a4cc685b2e4e57024d787 2024-11-20T05:15:42,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/1b9d9a9c355d4776a7823b12735dbe85 is 50, key is test_row_0/B:col10/1732079740653/Put/seqid=0 2024-11-20T05:15:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742080_1256 (size=12151) 2024-11-20T05:15:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079802137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079802137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:42,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079802137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:42,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53690 deadline: 1732079802357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:42,359 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in 63a460fa7bbfa3a642b55ff183f6f407 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:15:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079802448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079802448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079802448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:42,498 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/1b9d9a9c355d4776a7823b12735dbe85 2024-11-20T05:15:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/eb494ff8960443ecad75566a7d652c03 is 50, key is test_row_0/C:col10/1732079740653/Put/seqid=0 2024-11-20T05:15:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742081_1257 (size=12151) 2024-11-20T05:15:42,916 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/eb494ff8960443ecad75566a7d652c03 2024-11-20T05:15:42,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/dd237485e03a4cc685b2e4e57024d787 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/dd237485e03a4cc685b2e4e57024d787 2024-11-20T05:15:42,925 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/dd237485e03a4cc685b2e4e57024d787, entries=150, sequenceid=227, filesize=11.9 K 2024-11-20T05:15:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/1b9d9a9c355d4776a7823b12735dbe85 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1b9d9a9c355d4776a7823b12735dbe85 2024-11-20T05:15:42,929 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1b9d9a9c355d4776a7823b12735dbe85, entries=150, sequenceid=227, filesize=11.9 K 2024-11-20T05:15:42,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/eb494ff8960443ecad75566a7d652c03 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/eb494ff8960443ecad75566a7d652c03 2024-11-20T05:15:42,934 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/eb494ff8960443ecad75566a7d652c03, entries=150, sequenceid=227, filesize=11.9 K 2024-11-20T05:15:42,935 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 63a460fa7bbfa3a642b55ff183f6f407 in 1311ms, sequenceid=227, compaction requested=false 2024-11-20T05:15:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-20T05:15:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-20T05:15:42,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-20T05:15:42,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0760 sec 2024-11-20T05:15:42,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.0800 sec 2024-11-20T05:15:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:42,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T05:15:42,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:42,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:42,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/86b2b5afc67e42779a3fca222a5ec7d1 is 50, key is test_row_1/A:col10/1732079741813/Put/seqid=0 2024-11-20T05:15:42,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742082_1258 (size=14537) 2024-11-20T05:15:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T05:15:43,007 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:43,008 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-20T05:15:43,009 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T05:15:43,010 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:43,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079803014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079803014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079803015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079803016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T05:15:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079803128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079803128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079803128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079803128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T05:15:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T05:15:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T05:15:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079803337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079803337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079803337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079803337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/86b2b5afc67e42779a3fca222a5ec7d1 2024-11-20T05:15:43,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/23235d29a36c4d9780ff8b07e3996cba is 50, key is test_row_1/B:col10/1732079741813/Put/seqid=0 2024-11-20T05:15:43,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742083_1259 (size=9757) 2024-11-20T05:15:43,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/23235d29a36c4d9780ff8b07e3996cba 2024-11-20T05:15:43,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/b15853e2693041838f1d31d4cb3e98a5 is 50, key is test_row_1/C:col10/1732079741813/Put/seqid=0 2024-11-20T05:15:43,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742084_1260 (size=9757) 2024-11-20T05:15:43,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/b15853e2693041838f1d31d4cb3e98a5 2024-11-20T05:15:43,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T05:15:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/86b2b5afc67e42779a3fca222a5ec7d1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/86b2b5afc67e42779a3fca222a5ec7d1 2024-11-20T05:15:43,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/86b2b5afc67e42779a3fca222a5ec7d1, entries=200, sequenceid=240, filesize=14.2 K 2024-11-20T05:15:43,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/23235d29a36c4d9780ff8b07e3996cba as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/23235d29a36c4d9780ff8b07e3996cba 2024-11-20T05:15:43,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/23235d29a36c4d9780ff8b07e3996cba, entries=100, sequenceid=240, filesize=9.5 K 2024-11-20T05:15:43,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/b15853e2693041838f1d31d4cb3e98a5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b15853e2693041838f1d31d4cb3e98a5 2024-11-20T05:15:43,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b15853e2693041838f1d31d4cb3e98a5, entries=100, sequenceid=240, filesize=9.5 K 2024-11-20T05:15:43,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 63a460fa7bbfa3a642b55ff183f6f407 in 544ms, sequenceid=240, compaction requested=true 2024-11-20T05:15:43,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:43,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:43,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:43,503 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:43,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:43,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:43,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:43,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T05:15:43,504 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:43,505 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:43,505 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:43,505 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,505 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/cc5175151ccc48c6b8988c2cb4a0b948, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/dd237485e03a4cc685b2e4e57024d787, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/86b2b5afc67e42779a3fca222a5ec7d1] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=38.4 K 2024-11-20T05:15:43,505 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:43,505 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:43,505 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,505 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/5fee276283d749c9a4e021fc0abead78, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1b9d9a9c355d4776a7823b12735dbe85, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/23235d29a36c4d9780ff8b07e3996cba] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=33.7 K 2024-11-20T05:15:43,505 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5175151ccc48c6b8988c2cb4a0b948, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732079739960 2024-11-20T05:15:43,506 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fee276283d749c9a4e021fc0abead78, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732079739960 2024-11-20T05:15:43,506 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd237485e03a4cc685b2e4e57024d787, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732079740653 2024-11-20T05:15:43,506 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9d9a9c355d4776a7823b12735dbe85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732079740653 2024-11-20T05:15:43,506 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86b2b5afc67e42779a3fca222a5ec7d1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732079741807 2024-11-20T05:15:43,506 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 23235d29a36c4d9780ff8b07e3996cba, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732079741813 2024-11-20T05:15:43,524 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:43,524 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/f75cddd456af40de89aee72f17923665 is 50, key is test_row_0/A:col10/1732079740653/Put/seqid=0 2024-11-20T05:15:43,536 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:43,537 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/28c59f920c56481c81d20e6751f32346 is 50, key is test_row_0/B:col10/1732079740653/Put/seqid=0 2024-11-20T05:15:43,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742085_1261 (size=12697) 2024-11-20T05:15:43,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742086_1262 (size=12697) 2024-11-20T05:15:43,619 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/28c59f920c56481c81d20e6751f32346 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/28c59f920c56481c81d20e6751f32346 2024-11-20T05:15:43,625 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into 28c59f920c56481c81d20e6751f32346(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:43,625 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:43,625 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=13, startTime=1732079743503; duration=0sec 2024-11-20T05:15:43,625 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:43,625 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:43,625 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:43,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T05:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,633 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:43,633 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:43,633 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:43,633 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4c82757075bf42d2a7f6e921fd193e99, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/eb494ff8960443ecad75566a7d652c03, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b15853e2693041838f1d31d4cb3e98a5] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=33.7 K 2024-11-20T05:15:43,634 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T05:15:43,634 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c82757075bf42d2a7f6e921fd193e99, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732079739960 2024-11-20T05:15:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:43,635 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting eb494ff8960443ecad75566a7d652c03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732079740653 2024-11-20T05:15:43,635 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b15853e2693041838f1d31d4cb3e98a5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732079741813 2024-11-20T05:15:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T05:15:43,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/22f1c444f98b4deeb7113bfee0445d8e is 50, key is test_row_0/A:col10/1732079743014/Put/seqid=0 2024-11-20T05:15:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:43,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:43,661 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:43,661 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/1827b8b422fb40af86372ed44efdaa3d is 50, key is test_row_0/C:col10/1732079740653/Put/seqid=0 2024-11-20T05:15:43,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742088_1264 (size=12697) 2024-11-20T05:15:43,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079803681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079803682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079803692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079803693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742087_1263 (size=12201) 2024-11-20T05:15:43,789 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T05:15:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079803797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079803797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079803807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:43,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079803808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,003 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/f75cddd456af40de89aee72f17923665 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f75cddd456af40de89aee72f17923665 2024-11-20T05:15:44,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079804007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,011 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into f75cddd456af40de89aee72f17923665(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:44,011 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:44,011 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=13, startTime=1732079743503; duration=0sec 2024-11-20T05:15:44,011 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:44,011 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079804007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079804017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079804018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,098 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/1827b8b422fb40af86372ed44efdaa3d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/1827b8b422fb40af86372ed44efdaa3d 2024-11-20T05:15:44,103 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/22f1c444f98b4deeb7113bfee0445d8e 2024-11-20T05:15:44,108 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into 1827b8b422fb40af86372ed44efdaa3d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:44,108 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:44,108 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=13, startTime=1732079743503; duration=0sec 2024-11-20T05:15:44,108 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:44,108 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:44,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/35881320440d42a697273782cf9f2bf3 is 50, key is test_row_0/B:col10/1732079743014/Put/seqid=0 2024-11-20T05:15:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T05:15:44,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742089_1265 (size=12201) 2024-11-20T05:15:44,162 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/35881320440d42a697273782cf9f2bf3 2024-11-20T05:15:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/64770d6c23ec40e99e3c2b2e048ea834 is 50, key is test_row_0/C:col10/1732079743014/Put/seqid=0 2024-11-20T05:15:44,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742090_1266 (size=12201) 2024-11-20T05:15:44,203 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/64770d6c23ec40e99e3c2b2e048ea834 2024-11-20T05:15:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/22f1c444f98b4deeb7113bfee0445d8e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/22f1c444f98b4deeb7113bfee0445d8e 2024-11-20T05:15:44,215 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/22f1c444f98b4deeb7113bfee0445d8e, entries=150, sequenceid=260, filesize=11.9 K 2024-11-20T05:15:44,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/35881320440d42a697273782cf9f2bf3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/35881320440d42a697273782cf9f2bf3 2024-11-20T05:15:44,220 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/35881320440d42a697273782cf9f2bf3, entries=150, sequenceid=260, filesize=11.9 K 2024-11-20T05:15:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/64770d6c23ec40e99e3c2b2e048ea834 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/64770d6c23ec40e99e3c2b2e048ea834 2024-11-20T05:15:44,228 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/64770d6c23ec40e99e3c2b2e048ea834, entries=150, sequenceid=260, filesize=11.9 K 2024-11-20T05:15:44,229 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 63a460fa7bbfa3a642b55ff183f6f407 in 596ms, sequenceid=260, compaction requested=false 2024-11-20T05:15:44,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:44,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:44,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-20T05:15:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-20T05:15:44,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T05:15:44,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2210 sec 2024-11-20T05:15:44,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2290 sec 2024-11-20T05:15:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:44,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T05:15:44,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:44,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:44,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:44,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:44,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:44,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:44,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/4f904350ad1d4b29b83a91e79e2b65c5 is 50, key is test_row_0/A:col10/1732079743690/Put/seqid=0 2024-11-20T05:15:44,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742091_1267 (size=12301) 2024-11-20T05:15:44,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/4f904350ad1d4b29b83a91e79e2b65c5 2024-11-20T05:15:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079804353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/1c9038f1565747618fd34d551746f898 is 50, key is test_row_0/B:col10/1732079743690/Put/seqid=0 2024-11-20T05:15:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079804360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079804366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079804366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742092_1268 (size=12301) 2024-11-20T05:15:44,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/1c9038f1565747618fd34d551746f898 2024-11-20T05:15:44,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/fdc6c700abec42c5a92bc861a4659106 is 50, key is test_row_0/C:col10/1732079743690/Put/seqid=0 2024-11-20T05:15:44,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742093_1269 (size=12301) 2024-11-20T05:15:44,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/fdc6c700abec42c5a92bc861a4659106 2024-11-20T05:15:44,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/4f904350ad1d4b29b83a91e79e2b65c5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/4f904350ad1d4b29b83a91e79e2b65c5 2024-11-20T05:15:44,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/4f904350ad1d4b29b83a91e79e2b65c5, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T05:15:44,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/1c9038f1565747618fd34d551746f898 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1c9038f1565747618fd34d551746f898 2024-11-20T05:15:44,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1c9038f1565747618fd34d551746f898, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T05:15:44,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/fdc6c700abec42c5a92bc861a4659106 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/fdc6c700abec42c5a92bc861a4659106 2024-11-20T05:15:44,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/fdc6c700abec42c5a92bc861a4659106, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T05:15:44,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 63a460fa7bbfa3a642b55ff183f6f407 in 155ms, sequenceid=277, compaction requested=true 2024-11-20T05:15:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:44,476 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T05:15:44,477 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:44,477 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:44,477 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:44,478 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:44,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T05:15:44,478 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f75cddd456af40de89aee72f17923665, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/22f1c444f98b4deeb7113bfee0445d8e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/4f904350ad1d4b29b83a91e79e2b65c5] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=36.3 K 2024-11-20T05:15:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:44,479 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:44,479 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f75cddd456af40de89aee72f17923665, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732079740653 2024-11-20T05:15:44,479 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:44,479 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:44,479 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/28c59f920c56481c81d20e6751f32346, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/35881320440d42a697273782cf9f2bf3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1c9038f1565747618fd34d551746f898] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=36.3 K 2024-11-20T05:15:44,480 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f1c444f98b4deeb7113bfee0445d8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732079742968 2024-11-20T05:15:44,481 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c59f920c56481c81d20e6751f32346, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732079740653 2024-11-20T05:15:44,481 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f904350ad1d4b29b83a91e79e2b65c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079743671 2024-11-20T05:15:44,481 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 35881320440d42a697273782cf9f2bf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732079742968 2024-11-20T05:15:44,481 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c9038f1565747618fd34d551746f898, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079743671 2024-11-20T05:15:44,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/ba0e90f6465148d8a00f45157667d092 is 50, key is test_row_0/A:col10/1732079744350/Put/seqid=0 2024-11-20T05:15:44,507 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:44,507 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/d697b76375374c5085a372d762c038c2 is 50, key is test_row_0/A:col10/1732079743690/Put/seqid=0 2024-11-20T05:15:44,516 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:44,517 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/feea70aec29644f3a42773958a55a684 is 50, key is test_row_0/B:col10/1732079743690/Put/seqid=0 2024-11-20T05:15:44,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079804503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079804508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079804505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079804517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742094_1270 (size=14741) 2024-11-20T05:15:44,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/ba0e90f6465148d8a00f45157667d092 2024-11-20T05:15:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742095_1271 (size=12949) 2024-11-20T05:15:44,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/dff02b7a7054403b95ca92e6e9d96da6 is 50, key is test_row_0/B:col10/1732079744350/Put/seqid=0 2024-11-20T05:15:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742096_1272 (size=12949) 2024-11-20T05:15:44,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079804628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079804628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079804628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079804628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742097_1273 (size=12301) 2024-11-20T05:15:44,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/dff02b7a7054403b95ca92e6e9d96da6 2024-11-20T05:15:44,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/79f9e29f69c649f383278324ec84a900 is 50, key is test_row_0/C:col10/1732079744350/Put/seqid=0 2024-11-20T05:15:44,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742098_1274 (size=12301) 2024-11-20T05:15:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079804837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079804837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079804837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079804837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,006 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/d697b76375374c5085a372d762c038c2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d697b76375374c5085a372d762c038c2 2024-11-20T05:15:45,012 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into d697b76375374c5085a372d762c038c2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:45,012 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:45,012 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=13, startTime=1732079744476; duration=0sec 2024-11-20T05:15:45,012 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:45,012 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:45,012 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:45,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:45,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:45,014 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:45,014 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/1827b8b422fb40af86372ed44efdaa3d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/64770d6c23ec40e99e3c2b2e048ea834, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/fdc6c700abec42c5a92bc861a4659106] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=36.3 K 2024-11-20T05:15:45,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1827b8b422fb40af86372ed44efdaa3d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732079740653 2024-11-20T05:15:45,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64770d6c23ec40e99e3c2b2e048ea834, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732079742968 2024-11-20T05:15:45,015 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc6c700abec42c5a92bc861a4659106, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079743671 2024-11-20T05:15:45,023 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:45,023 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/ffe380ceb20e4745a10b6d2761abcad7 is 50, key is test_row_0/C:col10/1732079743690/Put/seqid=0 2024-11-20T05:15:45,030 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/feea70aec29644f3a42773958a55a684 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/feea70aec29644f3a42773958a55a684 2024-11-20T05:15:45,038 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into feea70aec29644f3a42773958a55a684(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:45,038 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:45,038 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=13, startTime=1732079744476; duration=0sec 2024-11-20T05:15:45,038 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:45,038 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742099_1275 (size=12949) 2024-11-20T05:15:45,086 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/ffe380ceb20e4745a10b6d2761abcad7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ffe380ceb20e4745a10b6d2761abcad7 2024-11-20T05:15:45,092 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into ffe380ceb20e4745a10b6d2761abcad7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:45,092 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:45,092 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=13, startTime=1732079744476; duration=0sec 2024-11-20T05:15:45,092 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:45,092 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:45,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/79f9e29f69c649f383278324ec84a900 2024-11-20T05:15:45,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/ba0e90f6465148d8a00f45157667d092 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ba0e90f6465148d8a00f45157667d092 2024-11-20T05:15:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079805147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079805147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079805147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079805147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ba0e90f6465148d8a00f45157667d092, entries=200, sequenceid=296, filesize=14.4 K 2024-11-20T05:15:45,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/dff02b7a7054403b95ca92e6e9d96da6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/dff02b7a7054403b95ca92e6e9d96da6 2024-11-20T05:15:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T05:15:45,158 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:45,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T05:15:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T05:15:45,162 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:45,163 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:45,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:45,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/dff02b7a7054403b95ca92e6e9d96da6, entries=150, sequenceid=296, filesize=12.0 K 2024-11-20T05:15:45,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/79f9e29f69c649f383278324ec84a900 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/79f9e29f69c649f383278324ec84a900 2024-11-20T05:15:45,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/79f9e29f69c649f383278324ec84a900, entries=150, sequenceid=296, filesize=12.0 K 2024-11-20T05:15:45,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 63a460fa7bbfa3a642b55ff183f6f407 in 702ms, sequenceid=296, compaction requested=false 2024-11-20T05:15:45,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T05:15:45,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T05:15:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:45,318 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T05:15:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:45,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/6b27a1c091144378842410b6ec79e2f5 is 50, key is test_row_0/A:col10/1732079744500/Put/seqid=0 2024-11-20T05:15:45,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742100_1276 (size=12301) 2024-11-20T05:15:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T05:15:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:45,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:45,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079805683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079805683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079805684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079805687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,743 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/6b27a1c091144378842410b6ec79e2f5 2024-11-20T05:15:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/c026c148de7b47d5bfa12a3253ba99bb is 50, key is test_row_0/B:col10/1732079744500/Put/seqid=0 2024-11-20T05:15:45,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742101_1277 (size=12301) 2024-11-20T05:15:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T05:15:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079805797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079805798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079805798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079805798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:45,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:45,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079806008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079806008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079806008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079806008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,168 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/c026c148de7b47d5bfa12a3253ba99bb 2024-11-20T05:15:46,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/6b4dee55e0c848c3b81fedcbee3e8360 is 50, key is test_row_0/C:col10/1732079744500/Put/seqid=0 2024-11-20T05:15:46,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742102_1278 (size=12301) 2024-11-20T05:15:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T05:15:46,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079806317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079806317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079806317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079806317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,579 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/6b4dee55e0c848c3b81fedcbee3e8360 2024-11-20T05:15:46,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/6b27a1c091144378842410b6ec79e2f5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/6b27a1c091144378842410b6ec79e2f5 2024-11-20T05:15:46,590 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/6b27a1c091144378842410b6ec79e2f5, entries=150, sequenceid=312, filesize=12.0 K 2024-11-20T05:15:46,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/c026c148de7b47d5bfa12a3253ba99bb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c026c148de7b47d5bfa12a3253ba99bb 2024-11-20T05:15:46,607 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c026c148de7b47d5bfa12a3253ba99bb, entries=150, sequenceid=312, filesize=12.0 K 2024-11-20T05:15:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/6b4dee55e0c848c3b81fedcbee3e8360 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6b4dee55e0c848c3b81fedcbee3e8360 2024-11-20T05:15:46,612 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6b4dee55e0c848c3b81fedcbee3e8360, entries=150, sequenceid=312, filesize=12.0 K 2024-11-20T05:15:46,613 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 63a460fa7bbfa3a642b55ff183f6f407 in 1295ms, sequenceid=312, compaction requested=true 2024-11-20T05:15:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T05:15:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T05:15:46,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T05:15:46,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4510 sec 2024-11-20T05:15:46,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4600 sec 2024-11-20T05:15:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:46,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:15:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:46,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/c890adb0ca744779b18f7f71da19c4b8 is 50, key is test_row_0/A:col10/1732079745675/Put/seqid=0 2024-11-20T05:15:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079806843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079806845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079806850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079806851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742103_1279 (size=12301) 2024-11-20T05:15:46,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/c890adb0ca744779b18f7f71da19c4b8 2024-11-20T05:15:46,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/4e8509ee4d894202b2675375f175fa62 is 50, key is test_row_0/B:col10/1732079745675/Put/seqid=0 2024-11-20T05:15:46,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742104_1280 (size=12301) 2024-11-20T05:15:46,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079806958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079806958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079806967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079806967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079807168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079807168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079807177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079807178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T05:15:47,307 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:47,308 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T05:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T05:15:47,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:47,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:47,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:47,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/4e8509ee4d894202b2675375f175fa62 2024-11-20T05:15:47,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/407f069c4d5d4fd89a231786f612f642 is 50, key is test_row_0/C:col10/1732079745675/Put/seqid=0 2024-11-20T05:15:47,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742105_1281 (size=12301) 2024-11-20T05:15:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T05:15:47,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T05:15:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:47,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:47,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:47,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079807477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079807477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079807487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079807487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:47,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T05:15:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:47,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T05:15:47,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/407f069c4d5d4fd89a231786f612f642 2024-11-20T05:15:47,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/c890adb0ca744779b18f7f71da19c4b8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/c890adb0ca744779b18f7f71da19c4b8 2024-11-20T05:15:47,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/c890adb0ca744779b18f7f71da19c4b8, entries=150, sequenceid=332, filesize=12.0 K 2024-11-20T05:15:47,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/4e8509ee4d894202b2675375f175fa62 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4e8509ee4d894202b2675375f175fa62 2024-11-20T05:15:47,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4e8509ee4d894202b2675375f175fa62, entries=150, sequenceid=332, filesize=12.0 K 2024-11-20T05:15:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/407f069c4d5d4fd89a231786f612f642 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/407f069c4d5d4fd89a231786f612f642 2024-11-20T05:15:47,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/407f069c4d5d4fd89a231786f612f642, entries=150, sequenceid=332, filesize=12.0 K 2024-11-20T05:15:47,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63a460fa7bbfa3a642b55ff183f6f407 in 938ms, sequenceid=332, compaction requested=true 2024-11-20T05:15:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:47,769 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:47,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T05:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:47,770 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T05:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:47,770 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:47,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:47,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:47,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:47,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:47,774 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:47,775 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:47,775 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:47,775 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d697b76375374c5085a372d762c038c2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ba0e90f6465148d8a00f45157667d092, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/6b27a1c091144378842410b6ec79e2f5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/c890adb0ca744779b18f7f71da19c4b8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=51.1 K 2024-11-20T05:15:47,775 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:47,775 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:47,776 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:47,776 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/feea70aec29644f3a42773958a55a684, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/dff02b7a7054403b95ca92e6e9d96da6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c026c148de7b47d5bfa12a3253ba99bb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4e8509ee4d894202b2675375f175fa62] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=48.7 K 2024-11-20T05:15:47,776 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d697b76375374c5085a372d762c038c2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079743671 2024-11-20T05:15:47,777 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting feea70aec29644f3a42773958a55a684, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079743671 2024-11-20T05:15:47,777 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0e90f6465148d8a00f45157667d092, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732079744350 2024-11-20T05:15:47,777 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting dff02b7a7054403b95ca92e6e9d96da6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732079744350 2024-11-20T05:15:47,778 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c026c148de7b47d5bfa12a3253ba99bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732079744500 2024-11-20T05:15:47,778 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b27a1c091144378842410b6ec79e2f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732079744500 2024-11-20T05:15:47,778 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e8509ee4d894202b2675375f175fa62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732079745675 2024-11-20T05:15:47,779 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c890adb0ca744779b18f7f71da19c4b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732079745675 2024-11-20T05:15:47,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/b999cd7f1bde402980cde0fbd4fb8737 is 50, key is test_row_1/A:col10/1732079746849/Put/seqid=0 2024-11-20T05:15:47,799 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:47,799 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/aea6d201205449dda522b6233ca03a76 is 50, key is test_row_0/B:col10/1732079745675/Put/seqid=0 2024-11-20T05:15:47,806 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:47,807 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/0df43363218d4003abe1c45e45618e38 is 50, key is test_row_0/A:col10/1732079745675/Put/seqid=0 2024-11-20T05:15:47,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742106_1282 (size=9857) 2024-11-20T05:15:47,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742107_1283 (size=13085) 2024-11-20T05:15:47,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742108_1284 (size=13085) 2024-11-20T05:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T05:15:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:48,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079808018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079808019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079748010, max timestamp=1732079748024Num uniques 859; 2024-11-20T05:15:48,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:48,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079748010, max timestamp=1732079748024Num uniques 859; 2024-11-20T05:15:48,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:48,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079748010, max timestamp=1732079748024Num uniques 859; 2024-11-20T05:15:48,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:15:48,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079808073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079808080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079808127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079808137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079808188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079808197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,230 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/b999cd7f1bde402980cde0fbd4fb8737 2024-11-20T05:15:48,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b5733bf1876c4aa183747755715281a6 is 50, key is test_row_1/B:col10/1732079746849/Put/seqid=0 2024-11-20T05:15:48,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742109_1285 (size=9857) 2024-11-20T05:15:48,279 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/aea6d201205449dda522b6233ca03a76 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/aea6d201205449dda522b6233ca03a76 2024-11-20T05:15:48,284 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into aea6d201205449dda522b6233ca03a76(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:48,284 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:48,284 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=12, startTime=1732079747770; duration=0sec 2024-11-20T05:15:48,284 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:48,284 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:48,284 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:15:48,286 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:15:48,286 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:48,286 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:48,286 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ffe380ceb20e4745a10b6d2761abcad7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/79f9e29f69c649f383278324ec84a900, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6b4dee55e0c848c3b81fedcbee3e8360, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/407f069c4d5d4fd89a231786f612f642] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=48.7 K 2024-11-20T05:15:48,287 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe380ceb20e4745a10b6d2761abcad7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079743671 2024-11-20T05:15:48,287 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f9e29f69c649f383278324ec84a900, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732079744350 2024-11-20T05:15:48,287 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b4dee55e0c848c3b81fedcbee3e8360, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732079744500 2024-11-20T05:15:48,288 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 407f069c4d5d4fd89a231786f612f642, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732079745675 2024-11-20T05:15:48,295 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/0df43363218d4003abe1c45e45618e38 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0df43363218d4003abe1c45e45618e38 2024-11-20T05:15:48,303 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:48,303 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/ee686ae05b7f4822ba9b9f00a6464a72 is 50, key is test_row_0/C:col10/1732079745675/Put/seqid=0 2024-11-20T05:15:48,303 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into 0df43363218d4003abe1c45e45618e38(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:48,303 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:48,303 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=12, startTime=1732079747769; duration=0sec 2024-11-20T05:15:48,304 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:48,304 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:48,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742110_1286 (size=13085) 2024-11-20T05:15:48,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079808337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079808347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079808400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079808407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T05:15:48,646 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b5733bf1876c4aa183747755715281a6 2024-11-20T05:15:48,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079808648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079808658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a3060c28f27245c3a11efeea75d0486b is 50, key is test_row_1/C:col10/1732079746849/Put/seqid=0 2024-11-20T05:15:48,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742111_1287 (size=9857) 2024-11-20T05:15:48,682 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a3060c28f27245c3a11efeea75d0486b 2024-11-20T05:15:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/b999cd7f1bde402980cde0fbd4fb8737 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b999cd7f1bde402980cde0fbd4fb8737 2024-11-20T05:15:48,701 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b999cd7f1bde402980cde0fbd4fb8737, entries=100, sequenceid=344, filesize=9.6 K 2024-11-20T05:15:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b5733bf1876c4aa183747755715281a6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b5733bf1876c4aa183747755715281a6 2024-11-20T05:15:48,710 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b5733bf1876c4aa183747755715281a6, entries=100, sequenceid=344, filesize=9.6 K 2024-11-20T05:15:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a3060c28f27245c3a11efeea75d0486b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a3060c28f27245c3a11efeea75d0486b 2024-11-20T05:15:48,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079808707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079808717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:48,768 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/ee686ae05b7f4822ba9b9f00a6464a72 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ee686ae05b7f4822ba9b9f00a6464a72 2024-11-20T05:15:48,769 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a3060c28f27245c3a11efeea75d0486b, entries=100, sequenceid=344, filesize=9.6 K 2024-11-20T05:15:48,770 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for 63a460fa7bbfa3a642b55ff183f6f407 in 1000ms, sequenceid=344, compaction requested=false 2024-11-20T05:15:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T05:15:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T05:15:48,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T05:15:48,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-11-20T05:15:48,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4700 sec 2024-11-20T05:15:48,780 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into ee686ae05b7f4822ba9b9f00a6464a72(size=12.8 K), total size for store is 22.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:48,780 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:48,780 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=12, startTime=1732079747771; duration=0sec 2024-11-20T05:15:48,780 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:48,780 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:49,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-20T05:15:49,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:49,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:49,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:49,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:49,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:49,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:49,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/28735291c7af4878ba0cb35f2ef6952b is 50, key is test_row_0/A:col10/1732079748061/Put/seqid=0 2024-11-20T05:15:49,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742112_1288 (size=14741) 2024-11-20T05:15:49,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079809185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079809192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079809227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079809227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079809297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079809307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T05:15:49,457 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:49,457 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T05:15:49,459 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T05:15:49,460 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:49,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:49,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079809507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079809518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T05:15:49,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/28735291c7af4878ba0cb35f2ef6952b 2024-11-20T05:15:49,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/289136d6a18f4ea1a55c8ed3bf03e1ef is 50, key is test_row_0/B:col10/1732079748061/Put/seqid=0 2024-11-20T05:15:49,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742113_1289 (size=12301) 2024-11-20T05:15:49,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/289136d6a18f4ea1a55c8ed3bf03e1ef 2024-11-20T05:15:49,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/d5cfb94bf943421ba6dcddf65b7a3dce is 50, key is test_row_0/C:col10/1732079748061/Put/seqid=0 2024-11-20T05:15:49,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742114_1290 (size=12301) 2024-11-20T05:15:49,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T05:15:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:49,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T05:15:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:49,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T05:15:49,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079809825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079809827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:49,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T05:15:49,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:49,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:49,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:49,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:49,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/d5cfb94bf943421ba6dcddf65b7a3dce 2024-11-20T05:15:49,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/28735291c7af4878ba0cb35f2ef6952b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/28735291c7af4878ba0cb35f2ef6952b 2024-11-20T05:15:50,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/28735291c7af4878ba0cb35f2ef6952b, entries=200, sequenceid=383, filesize=14.4 K 2024-11-20T05:15:50,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/289136d6a18f4ea1a55c8ed3bf03e1ef as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/289136d6a18f4ea1a55c8ed3bf03e1ef 2024-11-20T05:15:50,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/289136d6a18f4ea1a55c8ed3bf03e1ef, entries=150, sequenceid=383, filesize=12.0 K 2024-11-20T05:15:50,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/d5cfb94bf943421ba6dcddf65b7a3dce as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/d5cfb94bf943421ba6dcddf65b7a3dce 2024-11-20T05:15:50,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/d5cfb94bf943421ba6dcddf65b7a3dce, entries=150, sequenceid=383, filesize=12.0 K 2024-11-20T05:15:50,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=53.67 KB/54960 for 63a460fa7bbfa3a642b55ff183f6f407 in 849ms, sequenceid=383, compaction requested=true 2024-11-20T05:15:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:50,011 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:50,011 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:50,012 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:50,012 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:50,012 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:50,012 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/aea6d201205449dda522b6233ca03a76, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b5733bf1876c4aa183747755715281a6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/289136d6a18f4ea1a55c8ed3bf03e1ef] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=34.4 K 2024-11-20T05:15:50,012 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:50,012 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:50,012 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:50,012 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0df43363218d4003abe1c45e45618e38, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b999cd7f1bde402980cde0fbd4fb8737, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/28735291c7af4878ba0cb35f2ef6952b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=36.8 K 2024-11-20T05:15:50,013 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting aea6d201205449dda522b6233ca03a76, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732079745675 2024-11-20T05:15:50,013 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df43363218d4003abe1c45e45618e38, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732079745675 2024-11-20T05:15:50,013 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b5733bf1876c4aa183747755715281a6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732079746848 2024-11-20T05:15:50,013 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b999cd7f1bde402980cde0fbd4fb8737, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732079746848 2024-11-20T05:15:50,013 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 289136d6a18f4ea1a55c8ed3bf03e1ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732079748061 2024-11-20T05:15:50,013 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28735291c7af4878ba0cb35f2ef6952b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732079748061 2024-11-20T05:15:50,026 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:50,027 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/ebdcb3e8a3f94009844f6f4ba90a0150 is 50, key is test_row_0/B:col10/1732079748061/Put/seqid=0 2024-11-20T05:15:50,032 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:50,032 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/b08a9aee1fc341bd8cb490a6f4050d84 is 50, key is test_row_0/A:col10/1732079748061/Put/seqid=0 2024-11-20T05:15:50,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742115_1291 (size=13187) 2024-11-20T05:15:50,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742116_1292 (size=13187) 2024-11-20T05:15:50,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T05:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:50,074 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:50,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/0162d2fad6f44966b14d2a3a566c42ab is 50, key is test_row_0/A:col10/1732079749183/Put/seqid=0 2024-11-20T05:15:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T05:15:50,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742117_1293 (size=12301) 2024-11-20T05:15:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:50,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:50,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079750294, max timestamp=1732079750305Num uniques 859; 2024-11-20T05:15:50,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079750294, max timestamp=1732079750305Num uniques 859; 2024-11-20T05:15:50,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079750294, max timestamp=1732079750305Num uniques 859; 2024-11-20T05:15:50,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:50,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:50,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:50,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079810351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079810352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079810352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079810352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,444 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/ebdcb3e8a3f94009844f6f4ba90a0150 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/ebdcb3e8a3f94009844f6f4ba90a0150 2024-11-20T05:15:50,448 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into ebdcb3e8a3f94009844f6f4ba90a0150(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:50,448 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:50,448 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=13, startTime=1732079750011; duration=0sec 2024-11-20T05:15:50,448 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:50,448 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:50,448 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:50,449 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:50,449 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:50,449 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:50,450 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ee686ae05b7f4822ba9b9f00a6464a72, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a3060c28f27245c3a11efeea75d0486b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/d5cfb94bf943421ba6dcddf65b7a3dce] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=34.4 K 2024-11-20T05:15:50,450 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ee686ae05b7f4822ba9b9f00a6464a72, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732079745675 2024-11-20T05:15:50,450 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a3060c28f27245c3a11efeea75d0486b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732079746848 2024-11-20T05:15:50,450 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d5cfb94bf943421ba6dcddf65b7a3dce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732079748061 2024-11-20T05:15:50,455 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:50,456 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/5d5ba099faa54e8aafdc324c12a4d5bf is 50, key is test_row_0/C:col10/1732079748061/Put/seqid=0 2024-11-20T05:15:50,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742118_1294 (size=13187) 2024-11-20T05:15:50,463 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/b08a9aee1fc341bd8cb490a6f4050d84 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b08a9aee1fc341bd8cb490a6f4050d84 2024-11-20T05:15:50,467 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into b08a9aee1fc341bd8cb490a6f4050d84(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:50,467 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:50,467 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=13, startTime=1732079750011; duration=0sec 2024-11-20T05:15:50,467 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:50,467 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079810467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079810467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079810467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079810467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,510 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/0162d2fad6f44966b14d2a3a566c42ab 2024-11-20T05:15:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b323f556b94944a39bc36753d43ed075 is 50, key is test_row_0/B:col10/1732079749183/Put/seqid=0 2024-11-20T05:15:50,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742119_1295 (size=12301) 2024-11-20T05:15:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T05:15:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079810677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079810677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079810678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079810678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,866 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/5d5ba099faa54e8aafdc324c12a4d5bf as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5d5ba099faa54e8aafdc324c12a4d5bf 2024-11-20T05:15:50,871 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into 5d5ba099faa54e8aafdc324c12a4d5bf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:50,871 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:50,871 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=13, startTime=1732079750011; duration=0sec 2024-11-20T05:15:50,871 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:50,871 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:50,920 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b323f556b94944a39bc36753d43ed075 2024-11-20T05:15:50,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/0174cbc47fe24322905207d44ff256fe is 50, key is test_row_0/C:col10/1732079749183/Put/seqid=0 2024-11-20T05:15:50,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742120_1296 (size=12301) 2024-11-20T05:15:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732079810987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53764 deadline: 1732079810987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53730 deadline: 1732079810987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53748 deadline: 1732079810987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:15:51,330 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/0174cbc47fe24322905207d44ff256fe 2024-11-20T05:15:51,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/0162d2fad6f44966b14d2a3a566c42ab as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0162d2fad6f44966b14d2a3a566c42ab 2024-11-20T05:15:51,342 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0162d2fad6f44966b14d2a3a566c42ab, entries=150, sequenceid=394, filesize=12.0 K 2024-11-20T05:15:51,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/b323f556b94944a39bc36753d43ed075 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b323f556b94944a39bc36753d43ed075 2024-11-20T05:15:51,349 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b323f556b94944a39bc36753d43ed075, entries=150, sequenceid=394, filesize=12.0 K 2024-11-20T05:15:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/0174cbc47fe24322905207d44ff256fe as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0174cbc47fe24322905207d44ff256fe 2024-11-20T05:15:51,355 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0174cbc47fe24322905207d44ff256fe, entries=150, sequenceid=394, filesize=12.0 K 2024-11-20T05:15:51,356 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 63a460fa7bbfa3a642b55ff183f6f407 in 1282ms, sequenceid=394, compaction requested=false 2024-11-20T05:15:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T05:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T05:15:51,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T05:15:51,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-11-20T05:15:51,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.9040 sec 2024-11-20T05:15:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:51,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079750350, max timestamp=1732079751499Num uniques 1068; 2024-11-20T05:15:51,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T05:15:51,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=107.34 KB heapSize=302.36 KB 2024-11-20T05:15:51,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079750350, max timestamp=1732079751499Num uniques 1068; 2024-11-20T05:15:51,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079750350, max timestamp=1732079751499Num uniques 1068; 2024-11-20T05:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:51,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T05:15:51,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T05:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:51,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/efb7f8f3aa8d4c3f9067f785cf8b8611 is 50, key is test_row_0/A:col10/1732079750351/Put/seqid=0 2024-11-20T05:15:51,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742121_1297 (size=12301) 2024-11-20T05:15:51,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/efb7f8f3aa8d4c3f9067f785cf8b8611 2024-11-20T05:15:51,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/88b576378da74433938b1731a88efb05 is 50, key is test_row_0/B:col10/1732079750351/Put/seqid=0 2024-11-20T05:15:51,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742122_1298 (size=12301) 2024-11-20T05:15:51,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079751522, max timestamp=1732079751531Num uniques 859; 2024-11-20T05:15:51,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079751522, max timestamp=1732079751532Num uniques 859; 2024-11-20T05:15:51,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:51,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:51,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732079751522, max timestamp=1732079751532Num uniques 859; 2024-11-20T05:15:51,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T05:15:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732079751552, max timestamp=1732079751564Num uniques 1118; 2024-11-20T05:15:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-20T05:15:51,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732079751552, max timestamp=1732079751564Num uniques 1118; 2024-11-20T05:15:51,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732079751552, max timestamp=1732079751564Num uniques 1118; 2024-11-20T05:15:51,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-20T05:15:51,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-20T05:15:51,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732079751587, max timestamp=1732079751601Num uniques 1127; 2024-11-20T05:15:51,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732079751587, max timestamp=1732079751601Num uniques 1127; 2024-11-20T05:15:51,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-20T05:15:51,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-20T05:15:51,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732079751587, max timestamp=1732079751601Num uniques 1127; 2024-11-20T05:15:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T05:15:51,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-20T05:15:51,607 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:51,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T05:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T05:15:51,611 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:51,612 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:51,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:51,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732079751612, max timestamp=1732079751639Num uniques 1136; 2024-11-20T05:15:51,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-20T05:15:51,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732079751612, max timestamp=1732079751639Num uniques 1136; 2024-11-20T05:15:51,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-20T05:15:51,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732079751612, max timestamp=1732079751639Num uniques 1136; 2024-11-20T05:15:51,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-20T05:15:51,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079751668, max timestamp=1732079751678Num uniques 1045; 2024-11-20T05:15:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-20T05:15:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079751668, max timestamp=1732079751678Num uniques 1045; 2024-11-20T05:15:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079751668, max timestamp=1732079751678Num uniques 1045; 2024-11-20T05:15:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-20T05:15:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-20T05:15:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T05:15:51,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732079751698, max timestamp=1732079751718Num uniques 1104; 2024-11-20T05:15:51,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-20T05:15:51,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732079751698, max timestamp=1732079751718Num uniques 1104; 2024-11-20T05:15:51,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-20T05:15:51,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732079751698, max timestamp=1732079751718Num uniques 1104; 2024-11-20T05:15:51,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-20T05:15:51,751 INFO [Thread-1024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1024. 2024-11-20T05:15:51,751 DEBUG [Thread-1024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,751 DEBUG [Thread-1024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,752 DEBUG [Thread-1024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,752 INFO [Thread-1009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1009. 2024-11-20T05:15:51,752 DEBUG [Thread-1009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,752 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,752 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,754 INFO [Thread-1015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1015. 2024-11-20T05:15:51,754 DEBUG [Thread-1015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,754 DEBUG [Thread-1015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,754 DEBUG [Thread-1015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,756 INFO [Thread-1026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1026. 2024-11-20T05:15:51,756 DEBUG [Thread-1026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,756 DEBUG [Thread-1026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,756 DEBUG [Thread-1026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,757 INFO [Thread-1013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1013. 2024-11-20T05:15:51,757 DEBUG [Thread-1013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,757 DEBUG [Thread-1013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,757 DEBUG [Thread-1013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,758 INFO [Thread-1017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1017. 2024-11-20T05:15:51,758 DEBUG [Thread-1017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,758 DEBUG [Thread-1017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,758 DEBUG [Thread-1017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732079751746, max timestamp=1732079751754Num uniques 1113; 2024-11-20T05:15:51,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-20T05:15:51,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732079751746, max timestamp=1732079751754Num uniques 1113; 2024-11-20T05:15:51,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732079751746, max timestamp=1732079751754Num uniques 1113; 2024-11-20T05:15:51,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-20T05:15:51,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-20T05:15:51,760 INFO [Thread-1028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1028. 2024-11-20T05:15:51,760 DEBUG [Thread-1028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,760 DEBUG [Thread-1028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,760 DEBUG [Thread-1028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,762 INFO [Thread-1020 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1020. 2024-11-20T05:15:51,762 DEBUG [Thread-1020 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,762 DEBUG [Thread-1020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,762 DEBUG [Thread-1020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,763 INFO [Thread-1022 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1022. 2024-11-20T05:15:51,763 DEBUG [Thread-1022 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:51,763 DEBUG [Thread-1022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,763 DEBUG [Thread-1022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:51,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:51,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T05:15:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:51,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:51,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T05:15:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:51,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:51,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T05:15:51,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/88b576378da74433938b1731a88efb05 2024-11-20T05:15:51,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/b978e25575f74675988010b11ee03af9 is 50, key is test_row_0/C:col10/1732079750351/Put/seqid=0 2024-11-20T05:15:51,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742123_1299 (size=12301) 2024-11-20T05:15:52,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T05:15:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:52,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:52,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T05:15:52,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. as already flushing 2024-11-20T05:15:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T05:15:52,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/b978e25575f74675988010b11ee03af9 2024-11-20T05:15:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/efb7f8f3aa8d4c3f9067f785cf8b8611 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/efb7f8f3aa8d4c3f9067f785cf8b8611 2024-11-20T05:15:52,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/efb7f8f3aa8d4c3f9067f785cf8b8611, entries=150, sequenceid=435, filesize=12.0 K 2024-11-20T05:15:52,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/88b576378da74433938b1731a88efb05 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/88b576378da74433938b1731a88efb05 2024-11-20T05:15:52,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/88b576378da74433938b1731a88efb05, entries=150, sequenceid=435, filesize=12.0 K 2024-11-20T05:15:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/b978e25575f74675988010b11ee03af9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b978e25575f74675988010b11ee03af9 2024-11-20T05:15:52,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b978e25575f74675988010b11ee03af9, entries=150, sequenceid=435, filesize=12.0 K 2024-11-20T05:15:52,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=40.25 KB/41220 for 63a460fa7bbfa3a642b55ff183f6f407 in 866ms, sequenceid=435, compaction requested=true 2024-11-20T05:15:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:15:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:52,369 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:15:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:52,369 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63a460fa7bbfa3a642b55ff183f6f407:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:15:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:52,370 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:52,370 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:52,370 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/A is initiating minor compaction (all files) 2024-11-20T05:15:52,370 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/B is initiating minor compaction (all files) 2024-11-20T05:15:52,370 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/A in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,370 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/B in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,370 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/ebdcb3e8a3f94009844f6f4ba90a0150, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b323f556b94944a39bc36753d43ed075, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/88b576378da74433938b1731a88efb05] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=36.9 K 2024-11-20T05:15:52,370 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b08a9aee1fc341bd8cb490a6f4050d84, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0162d2fad6f44966b14d2a3a566c42ab, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/efb7f8f3aa8d4c3f9067f785cf8b8611] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=36.9 K 2024-11-20T05:15:52,371 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b08a9aee1fc341bd8cb490a6f4050d84, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732079748061 2024-11-20T05:15:52,371 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdcb3e8a3f94009844f6f4ba90a0150, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732079748061 2024-11-20T05:15:52,371 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0162d2fad6f44966b14d2a3a566c42ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732079749183 2024-11-20T05:15:52,371 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b323f556b94944a39bc36753d43ed075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732079749183 2024-11-20T05:15:52,371 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb7f8f3aa8d4c3f9067f785cf8b8611, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732079750350 2024-11-20T05:15:52,371 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b576378da74433938b1731a88efb05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732079750350 2024-11-20T05:15:52,378 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:52,379 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/4beba35a361f4708be5fce6dc6374f57 is 50, key is test_row_0/B:col10/1732079750351/Put/seqid=0 2024-11-20T05:15:52,381 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:52,382 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/5a36f7dace114a70a997a41d65fb9d67 is 50, key is test_row_0/A:col10/1732079750351/Put/seqid=0 2024-11-20T05:15:52,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742124_1300 (size=13289) 2024-11-20T05:15:52,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742125_1301 (size=13289) 2024-11-20T05:15:52,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T05:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,388 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=40.25 KB heapSize=101.02 KB 2024-11-20T05:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:15:52,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/31d6506ce74a4c20aa0db9e31fc99173 is 50, key is test_row_0/A:col10/1732079751754/Put/seqid=0 2024-11-20T05:15:52,397 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/5a36f7dace114a70a997a41d65fb9d67 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5a36f7dace114a70a997a41d65fb9d67 2024-11-20T05:15:52,397 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/4beba35a361f4708be5fce6dc6374f57 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4beba35a361f4708be5fce6dc6374f57 2024-11-20T05:15:52,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742126_1302 (size=12301) 2024-11-20T05:15:52,399 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/31d6506ce74a4c20aa0db9e31fc99173 2024-11-20T05:15:52,403 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/B of 63a460fa7bbfa3a642b55ff183f6f407 into 4beba35a361f4708be5fce6dc6374f57(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:52,403 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:52,403 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/A of 63a460fa7bbfa3a642b55ff183f6f407 into 5a36f7dace114a70a997a41d65fb9d67(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:52,403 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/B, priority=13, startTime=1732079752369; duration=0sec 2024-11-20T05:15:52,403 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:52,403 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/A, priority=13, startTime=1732079752369; duration=0sec 2024-11-20T05:15:52,404 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:52,404 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:15:52,404 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:B 2024-11-20T05:15:52,404 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:A 2024-11-20T05:15:52,404 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:15:52,406 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:15:52,406 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): 63a460fa7bbfa3a642b55ff183f6f407/C is initiating minor compaction (all files) 2024-11-20T05:15:52,406 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63a460fa7bbfa3a642b55ff183f6f407/C in TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:52,406 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5d5ba099faa54e8aafdc324c12a4d5bf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0174cbc47fe24322905207d44ff256fe, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b978e25575f74675988010b11ee03af9] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp, totalSize=36.9 K 2024-11-20T05:15:52,407 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d5ba099faa54e8aafdc324c12a4d5bf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732079748061 2024-11-20T05:15:52,408 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0174cbc47fe24322905207d44ff256fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732079749183 2024-11-20T05:15:52,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/be3e5abaa1294a68a3419064fcf8b018 is 50, key is test_row_0/B:col10/1732079751754/Put/seqid=0 2024-11-20T05:15:52,410 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b978e25575f74675988010b11ee03af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732079750350 2024-11-20T05:15:52,418 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63a460fa7bbfa3a642b55ff183f6f407#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:15:52,418 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a5d9d7c8e1fc48de8df851fe891593a9 is 50, key is test_row_0/C:col10/1732079750351/Put/seqid=0 2024-11-20T05:15:52,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742127_1303 (size=12301) 2024-11-20T05:15:52,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742128_1304 (size=13289) 2024-11-20T05:15:52,449 INFO [Thread-1011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1011. 2024-11-20T05:15:52,449 DEBUG [Thread-1011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:15:52,449 DEBUG [Thread-1011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:52,449 DEBUG [Thread-1011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:52,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T05:15:52,822 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/be3e5abaa1294a68a3419064fcf8b018 2024-11-20T05:15:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/f5a0071fd7874213b84409bca87a367d is 50, key is test_row_0/C:col10/1732079751754/Put/seqid=0 2024-11-20T05:15:52,836 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/a5d9d7c8e1fc48de8df851fe891593a9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5d9d7c8e1fc48de8df851fe891593a9 2024-11-20T05:15:52,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742129_1305 (size=12301) 2024-11-20T05:15:52,841 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63a460fa7bbfa3a642b55ff183f6f407/C of 63a460fa7bbfa3a642b55ff183f6f407 into a5d9d7c8e1fc48de8df851fe891593a9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:15:52,841 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:52,841 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., storeName=63a460fa7bbfa3a642b55ff183f6f407/C, priority=13, startTime=1732079752369; duration=0sec 2024-11-20T05:15:52,841 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:15:52,841 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63a460fa7bbfa3a642b55ff183f6f407:C 2024-11-20T05:15:53,239 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/f5a0071fd7874213b84409bca87a367d 2024-11-20T05:15:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/31d6506ce74a4c20aa0db9e31fc99173 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/31d6506ce74a4c20aa0db9e31fc99173 2024-11-20T05:15:53,253 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/31d6506ce74a4c20aa0db9e31fc99173, entries=150, sequenceid=560, filesize=12.0 K 2024-11-20T05:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/be3e5abaa1294a68a3419064fcf8b018 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/be3e5abaa1294a68a3419064fcf8b018 2024-11-20T05:15:53,259 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/be3e5abaa1294a68a3419064fcf8b018, entries=150, sequenceid=560, filesize=12.0 K 2024-11-20T05:15:53,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/f5a0071fd7874213b84409bca87a367d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f5a0071fd7874213b84409bca87a367d 2024-11-20T05:15:53,264 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f5a0071fd7874213b84409bca87a367d, entries=150, sequenceid=560, filesize=12.0 K 2024-11-20T05:15:53,265 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~100.97 KB/103392, currentSize=6.71 KB/6870 for 63a460fa7bbfa3a642b55ff183f6f407 in 876ms, sequenceid=560, compaction requested=false 2024-11-20T05:15:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 63a460fa7bbfa3a642b55ff183f6f407: 2024-11-20T05:15:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T05:15:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T05:15:53,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T05:15:53,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6540 sec 2024-11-20T05:15:53,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6610 sec 2024-11-20T05:15:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T05:15:53,757 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2831 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2673 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2717 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2835 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2659 2024-11-20T05:15:53,757 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T05:15:53,758 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:15:53,758 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:15:53,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:53,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:53,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:15:53,759 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T05:15:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T05:15:53,762 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079753762"}]},"ts":"1732079753762"} 2024-11-20T05:15:53,764 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T05:15:53,764 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T05:15:53,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:15:53,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, UNASSIGN}] 2024-11-20T05:15:53,768 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, UNASSIGN 2024-11-20T05:15:53,769 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=63a460fa7bbfa3a642b55ff183f6f407, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:53,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, UNASSIGN because future has completed 2024-11-20T05:15:53,770 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:15:53,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:15:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T05:15:53,923 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:53,923 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:15:53,923 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 63a460fa7bbfa3a642b55ff183f6f407, disabling compactions & flushes 2024-11-20T05:15:53,923 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:53,923 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:53,923 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. after waiting 0 ms 2024-11-20T05:15:53,923 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:53,923 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 63a460fa7bbfa3a642b55ff183f6f407 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-20T05:15:53,924 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=A 2024-11-20T05:15:53,924 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:53,924 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=B 2024-11-20T05:15:53,924 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:53,924 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63a460fa7bbfa3a642b55ff183f6f407, store=C 2024-11-20T05:15:53,924 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:53,928 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/8f443fdc806e4745941c6906f8119a18 is 50, key is test_row_0/A:col10/1732079752448/Put/seqid=0 2024-11-20T05:15:53,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742130_1306 (size=7415) 2024-11-20T05:15:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T05:15:54,332 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/8f443fdc806e4745941c6906f8119a18 2024-11-20T05:15:54,341 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/07cb58fa73f3419f8d46793269766198 is 50, key is test_row_0/B:col10/1732079752448/Put/seqid=0 2024-11-20T05:15:54,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742131_1307 (size=7415) 2024-11-20T05:15:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T05:15:54,747 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/07cb58fa73f3419f8d46793269766198 2024-11-20T05:15:54,756 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/28ef640d35a8422bb7c6c8772f83e6a4 is 50, key is test_row_0/C:col10/1732079752448/Put/seqid=0 2024-11-20T05:15:54,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742132_1308 (size=7415) 2024-11-20T05:15:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T05:15:55,161 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/28ef640d35a8422bb7c6c8772f83e6a4 2024-11-20T05:15:55,165 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/A/8f443fdc806e4745941c6906f8119a18 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/8f443fdc806e4745941c6906f8119a18 2024-11-20T05:15:55,168 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/8f443fdc806e4745941c6906f8119a18, entries=50, sequenceid=567, filesize=7.2 K 2024-11-20T05:15:55,169 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/B/07cb58fa73f3419f8d46793269766198 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/07cb58fa73f3419f8d46793269766198 2024-11-20T05:15:55,172 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/07cb58fa73f3419f8d46793269766198, entries=50, sequenceid=567, filesize=7.2 K 2024-11-20T05:15:55,173 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/.tmp/C/28ef640d35a8422bb7c6c8772f83e6a4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/28ef640d35a8422bb7c6c8772f83e6a4 2024-11-20T05:15:55,176 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/28ef640d35a8422bb7c6c8772f83e6a4, entries=50, sequenceid=567, filesize=7.2 K 2024-11-20T05:15:55,177 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 63a460fa7bbfa3a642b55ff183f6f407 in 1254ms, sequenceid=567, compaction requested=true 2024-11-20T05:15:55,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f85d5f088e1b4da7af323983d348e979, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/bc3951a11e974f829256f222b68dce2a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5e37c4eeaf574b13903ff159a1bdc40c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/21075a6f24e846f0bfbd75bd5d8fed0c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/7b4f5fd5203c4f889712726879f44e28, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b27b687432db476baf33fae1949f296a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/19b8da6210f840f3ab13d621d86bfbb3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d8c4ba22500840799c457d01dcae40cf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/3b14d97753cc4828bd227adee6492ad0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/79dad19435a74f56929634369809b157, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/56f516c4ea9746aa97eb1aac07e59e3a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/de2c9e97c42f4770935630d11e8900b2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ce6cf3f59f974d4ab58771980ef41494, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/cc5175151ccc48c6b8988c2cb4a0b948, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/dd237485e03a4cc685b2e4e57024d787, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/86b2b5afc67e42779a3fca222a5ec7d1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f75cddd456af40de89aee72f17923665, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/22f1c444f98b4deeb7113bfee0445d8e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d697b76375374c5085a372d762c038c2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/4f904350ad1d4b29b83a91e79e2b65c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ba0e90f6465148d8a00f45157667d092, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/6b27a1c091144378842410b6ec79e2f5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0df43363218d4003abe1c45e45618e38, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/c890adb0ca744779b18f7f71da19c4b8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b999cd7f1bde402980cde0fbd4fb8737, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/28735291c7af4878ba0cb35f2ef6952b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b08a9aee1fc341bd8cb490a6f4050d84, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0162d2fad6f44966b14d2a3a566c42ab, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/efb7f8f3aa8d4c3f9067f785cf8b8611] to archive 2024-11-20T05:15:55,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:15:55,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f85d5f088e1b4da7af323983d348e979 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f85d5f088e1b4da7af323983d348e979 2024-11-20T05:15:55,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/bc3951a11e974f829256f222b68dce2a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/bc3951a11e974f829256f222b68dce2a 2024-11-20T05:15:55,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5e37c4eeaf574b13903ff159a1bdc40c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5e37c4eeaf574b13903ff159a1bdc40c 2024-11-20T05:15:55,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/21075a6f24e846f0bfbd75bd5d8fed0c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/21075a6f24e846f0bfbd75bd5d8fed0c 2024-11-20T05:15:55,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/7b4f5fd5203c4f889712726879f44e28 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/7b4f5fd5203c4f889712726879f44e28 2024-11-20T05:15:55,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b27b687432db476baf33fae1949f296a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b27b687432db476baf33fae1949f296a 2024-11-20T05:15:55,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/19b8da6210f840f3ab13d621d86bfbb3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/19b8da6210f840f3ab13d621d86bfbb3 2024-11-20T05:15:55,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d8c4ba22500840799c457d01dcae40cf to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d8c4ba22500840799c457d01dcae40cf 2024-11-20T05:15:55,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/3b14d97753cc4828bd227adee6492ad0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/3b14d97753cc4828bd227adee6492ad0 2024-11-20T05:15:55,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/79dad19435a74f56929634369809b157 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/79dad19435a74f56929634369809b157 2024-11-20T05:15:55,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/56f516c4ea9746aa97eb1aac07e59e3a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/56f516c4ea9746aa97eb1aac07e59e3a 2024-11-20T05:15:55,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/de2c9e97c42f4770935630d11e8900b2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/de2c9e97c42f4770935630d11e8900b2 2024-11-20T05:15:55,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ce6cf3f59f974d4ab58771980ef41494 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ce6cf3f59f974d4ab58771980ef41494 2024-11-20T05:15:55,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/cc5175151ccc48c6b8988c2cb4a0b948 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/cc5175151ccc48c6b8988c2cb4a0b948 2024-11-20T05:15:55,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/dd237485e03a4cc685b2e4e57024d787 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/dd237485e03a4cc685b2e4e57024d787 2024-11-20T05:15:55,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/86b2b5afc67e42779a3fca222a5ec7d1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/86b2b5afc67e42779a3fca222a5ec7d1 2024-11-20T05:15:55,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f75cddd456af40de89aee72f17923665 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/f75cddd456af40de89aee72f17923665 2024-11-20T05:15:55,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/22f1c444f98b4deeb7113bfee0445d8e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/22f1c444f98b4deeb7113bfee0445d8e 2024-11-20T05:15:55,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d697b76375374c5085a372d762c038c2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/d697b76375374c5085a372d762c038c2 2024-11-20T05:15:55,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/4f904350ad1d4b29b83a91e79e2b65c5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/4f904350ad1d4b29b83a91e79e2b65c5 2024-11-20T05:15:55,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ba0e90f6465148d8a00f45157667d092 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/ba0e90f6465148d8a00f45157667d092 2024-11-20T05:15:55,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/6b27a1c091144378842410b6ec79e2f5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/6b27a1c091144378842410b6ec79e2f5 2024-11-20T05:15:55,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0df43363218d4003abe1c45e45618e38 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0df43363218d4003abe1c45e45618e38 2024-11-20T05:15:55,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/c890adb0ca744779b18f7f71da19c4b8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/c890adb0ca744779b18f7f71da19c4b8 2024-11-20T05:15:55,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b999cd7f1bde402980cde0fbd4fb8737 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b999cd7f1bde402980cde0fbd4fb8737 2024-11-20T05:15:55,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/28735291c7af4878ba0cb35f2ef6952b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/28735291c7af4878ba0cb35f2ef6952b 2024-11-20T05:15:55,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b08a9aee1fc341bd8cb490a6f4050d84 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/b08a9aee1fc341bd8cb490a6f4050d84 2024-11-20T05:15:55,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0162d2fad6f44966b14d2a3a566c42ab to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/0162d2fad6f44966b14d2a3a566c42ab 2024-11-20T05:15:55,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/efb7f8f3aa8d4c3f9067f785cf8b8611 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/efb7f8f3aa8d4c3f9067f785cf8b8611 2024-11-20T05:15:55,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3e9cac4149b14123bc7e1ff3aa4f7a18, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690ec6400afa4ff5afb83c27b28a7f8f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/a02ac55a86fc4dd9ae38775f34b85df4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/e070c70e722c4c6d94832721a8a3bae1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/738c2b25237b4d1191a1f4c1f15ba56f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/8b177c806d4b42509eeed7b041f373e7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3420aecf181b4c7ab84766566a7d63e8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690becbd4997472daec5744f0fe53154, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c451c7fc921a40c1bb51de346f5a0ccc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/03dac48d24404cb6bbb1d4eeb1cd8263, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/33a735afd8e6418fad1ed93d8c8981de, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/277086ec64854d0099e94ba8a6c9cf10, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/5fee276283d749c9a4e021fc0abead78, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b9fd459db27c46d185f6a5ab3bef1843, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1b9d9a9c355d4776a7823b12735dbe85, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/28c59f920c56481c81d20e6751f32346, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/23235d29a36c4d9780ff8b07e3996cba, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/35881320440d42a697273782cf9f2bf3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/feea70aec29644f3a42773958a55a684, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1c9038f1565747618fd34d551746f898, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/dff02b7a7054403b95ca92e6e9d96da6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c026c148de7b47d5bfa12a3253ba99bb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/aea6d201205449dda522b6233ca03a76, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4e8509ee4d894202b2675375f175fa62, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b5733bf1876c4aa183747755715281a6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/ebdcb3e8a3f94009844f6f4ba90a0150, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/289136d6a18f4ea1a55c8ed3bf03e1ef, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b323f556b94944a39bc36753d43ed075, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/88b576378da74433938b1731a88efb05] to archive 2024-11-20T05:15:55,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:15:55,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3e9cac4149b14123bc7e1ff3aa4f7a18 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3e9cac4149b14123bc7e1ff3aa4f7a18 2024-11-20T05:15:55,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690ec6400afa4ff5afb83c27b28a7f8f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690ec6400afa4ff5afb83c27b28a7f8f 2024-11-20T05:15:55,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/a02ac55a86fc4dd9ae38775f34b85df4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/a02ac55a86fc4dd9ae38775f34b85df4 2024-11-20T05:15:55,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/e070c70e722c4c6d94832721a8a3bae1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/e070c70e722c4c6d94832721a8a3bae1 2024-11-20T05:15:55,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/738c2b25237b4d1191a1f4c1f15ba56f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/738c2b25237b4d1191a1f4c1f15ba56f 2024-11-20T05:15:55,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/8b177c806d4b42509eeed7b041f373e7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/8b177c806d4b42509eeed7b041f373e7 2024-11-20T05:15:55,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3420aecf181b4c7ab84766566a7d63e8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/3420aecf181b4c7ab84766566a7d63e8 2024-11-20T05:15:55,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690becbd4997472daec5744f0fe53154 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/690becbd4997472daec5744f0fe53154 2024-11-20T05:15:55,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c451c7fc921a40c1bb51de346f5a0ccc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c451c7fc921a40c1bb51de346f5a0ccc 2024-11-20T05:15:55,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/03dac48d24404cb6bbb1d4eeb1cd8263 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/03dac48d24404cb6bbb1d4eeb1cd8263 2024-11-20T05:15:55,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/33a735afd8e6418fad1ed93d8c8981de to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/33a735afd8e6418fad1ed93d8c8981de 2024-11-20T05:15:55,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/277086ec64854d0099e94ba8a6c9cf10 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/277086ec64854d0099e94ba8a6c9cf10 2024-11-20T05:15:55,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/5fee276283d749c9a4e021fc0abead78 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/5fee276283d749c9a4e021fc0abead78 2024-11-20T05:15:55,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b9fd459db27c46d185f6a5ab3bef1843 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b9fd459db27c46d185f6a5ab3bef1843 2024-11-20T05:15:55,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1b9d9a9c355d4776a7823b12735dbe85 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1b9d9a9c355d4776a7823b12735dbe85 2024-11-20T05:15:55,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/28c59f920c56481c81d20e6751f32346 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/28c59f920c56481c81d20e6751f32346 2024-11-20T05:15:55,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/23235d29a36c4d9780ff8b07e3996cba to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/23235d29a36c4d9780ff8b07e3996cba 2024-11-20T05:15:55,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/35881320440d42a697273782cf9f2bf3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/35881320440d42a697273782cf9f2bf3 2024-11-20T05:15:55,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/feea70aec29644f3a42773958a55a684 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/feea70aec29644f3a42773958a55a684 2024-11-20T05:15:55,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1c9038f1565747618fd34d551746f898 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/1c9038f1565747618fd34d551746f898 2024-11-20T05:15:55,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/dff02b7a7054403b95ca92e6e9d96da6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/dff02b7a7054403b95ca92e6e9d96da6 2024-11-20T05:15:55,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c026c148de7b47d5bfa12a3253ba99bb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/c026c148de7b47d5bfa12a3253ba99bb 2024-11-20T05:15:55,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/aea6d201205449dda522b6233ca03a76 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/aea6d201205449dda522b6233ca03a76 2024-11-20T05:15:55,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4e8509ee4d894202b2675375f175fa62 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4e8509ee4d894202b2675375f175fa62 2024-11-20T05:15:55,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b5733bf1876c4aa183747755715281a6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b5733bf1876c4aa183747755715281a6 2024-11-20T05:15:55,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/ebdcb3e8a3f94009844f6f4ba90a0150 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/ebdcb3e8a3f94009844f6f4ba90a0150 2024-11-20T05:15:55,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/289136d6a18f4ea1a55c8ed3bf03e1ef to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/289136d6a18f4ea1a55c8ed3bf03e1ef 2024-11-20T05:15:55,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b323f556b94944a39bc36753d43ed075 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/b323f556b94944a39bc36753d43ed075 2024-11-20T05:15:55,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/88b576378da74433938b1731a88efb05 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/88b576378da74433938b1731a88efb05 2024-11-20T05:15:55,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0bd6b00cb5d0439ea1414440b980057d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/cc41df560234471493030f2cb643ca95, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4f536a7ce2cf4fc1b7d8898d55c19e7f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f8db95036a3b4b279da832b2d5905d93, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5edc40ada6d345cfab84faf6d29004ab, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6f5fafc6668c46adaf8de68ab2d335d7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/93616162a39e4affa15cab84c6174a3b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4d27221eaeab430eb3d14a1c4267dac5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5dc02f1d2284916becb707609cdf3a6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/027a8ee3f3204231ba5f935b23a6a9f8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/3a4672b9044e4ae19e628c4ef477a8e2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4e3d8771338a4b69a84ac7ebea1521d7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4c82757075bf42d2a7f6e921fd193e99, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/641bffa503aa4ca6962dde908383863c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/eb494ff8960443ecad75566a7d652c03, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/1827b8b422fb40af86372ed44efdaa3d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b15853e2693041838f1d31d4cb3e98a5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/64770d6c23ec40e99e3c2b2e048ea834, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ffe380ceb20e4745a10b6d2761abcad7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/fdc6c700abec42c5a92bc861a4659106, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/79f9e29f69c649f383278324ec84a900, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6b4dee55e0c848c3b81fedcbee3e8360, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ee686ae05b7f4822ba9b9f00a6464a72, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/407f069c4d5d4fd89a231786f612f642, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a3060c28f27245c3a11efeea75d0486b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5d5ba099faa54e8aafdc324c12a4d5bf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/d5cfb94bf943421ba6dcddf65b7a3dce, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0174cbc47fe24322905207d44ff256fe, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b978e25575f74675988010b11ee03af9] to archive 2024-11-20T05:15:55,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:15:55,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0bd6b00cb5d0439ea1414440b980057d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0bd6b00cb5d0439ea1414440b980057d 2024-11-20T05:15:55,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/cc41df560234471493030f2cb643ca95 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/cc41df560234471493030f2cb643ca95 2024-11-20T05:15:55,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4f536a7ce2cf4fc1b7d8898d55c19e7f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4f536a7ce2cf4fc1b7d8898d55c19e7f 2024-11-20T05:15:55,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f8db95036a3b4b279da832b2d5905d93 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f8db95036a3b4b279da832b2d5905d93 2024-11-20T05:15:55,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5edc40ada6d345cfab84faf6d29004ab to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5edc40ada6d345cfab84faf6d29004ab 2024-11-20T05:15:55,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6f5fafc6668c46adaf8de68ab2d335d7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6f5fafc6668c46adaf8de68ab2d335d7 2024-11-20T05:15:55,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/93616162a39e4affa15cab84c6174a3b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/93616162a39e4affa15cab84c6174a3b 2024-11-20T05:15:55,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4d27221eaeab430eb3d14a1c4267dac5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4d27221eaeab430eb3d14a1c4267dac5 2024-11-20T05:15:55,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5dc02f1d2284916becb707609cdf3a6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5dc02f1d2284916becb707609cdf3a6 2024-11-20T05:15:55,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/027a8ee3f3204231ba5f935b23a6a9f8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/027a8ee3f3204231ba5f935b23a6a9f8 2024-11-20T05:15:55,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/3a4672b9044e4ae19e628c4ef477a8e2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/3a4672b9044e4ae19e628c4ef477a8e2 2024-11-20T05:15:55,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4e3d8771338a4b69a84ac7ebea1521d7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4e3d8771338a4b69a84ac7ebea1521d7 2024-11-20T05:15:55,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4c82757075bf42d2a7f6e921fd193e99 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/4c82757075bf42d2a7f6e921fd193e99 2024-11-20T05:15:55,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/641bffa503aa4ca6962dde908383863c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/641bffa503aa4ca6962dde908383863c 2024-11-20T05:15:55,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/eb494ff8960443ecad75566a7d652c03 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/eb494ff8960443ecad75566a7d652c03 2024-11-20T05:15:55,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/1827b8b422fb40af86372ed44efdaa3d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/1827b8b422fb40af86372ed44efdaa3d 2024-11-20T05:15:55,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b15853e2693041838f1d31d4cb3e98a5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b15853e2693041838f1d31d4cb3e98a5 2024-11-20T05:15:55,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/64770d6c23ec40e99e3c2b2e048ea834 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/64770d6c23ec40e99e3c2b2e048ea834 2024-11-20T05:15:55,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ffe380ceb20e4745a10b6d2761abcad7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ffe380ceb20e4745a10b6d2761abcad7 2024-11-20T05:15:55,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/fdc6c700abec42c5a92bc861a4659106 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/fdc6c700abec42c5a92bc861a4659106 2024-11-20T05:15:55,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/79f9e29f69c649f383278324ec84a900 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/79f9e29f69c649f383278324ec84a900 2024-11-20T05:15:55,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6b4dee55e0c848c3b81fedcbee3e8360 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/6b4dee55e0c848c3b81fedcbee3e8360 2024-11-20T05:15:55,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ee686ae05b7f4822ba9b9f00a6464a72 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/ee686ae05b7f4822ba9b9f00a6464a72 2024-11-20T05:15:55,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/407f069c4d5d4fd89a231786f612f642 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/407f069c4d5d4fd89a231786f612f642 2024-11-20T05:15:55,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a3060c28f27245c3a11efeea75d0486b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a3060c28f27245c3a11efeea75d0486b 2024-11-20T05:15:55,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5d5ba099faa54e8aafdc324c12a4d5bf to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/5d5ba099faa54e8aafdc324c12a4d5bf 2024-11-20T05:15:55,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/d5cfb94bf943421ba6dcddf65b7a3dce to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/d5cfb94bf943421ba6dcddf65b7a3dce 2024-11-20T05:15:55,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0174cbc47fe24322905207d44ff256fe to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/0174cbc47fe24322905207d44ff256fe 2024-11-20T05:15:55,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b978e25575f74675988010b11ee03af9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/b978e25575f74675988010b11ee03af9 2024-11-20T05:15:55,267 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/recovered.edits/570.seqid, newMaxSeqId=570, maxSeqId=1 2024-11-20T05:15:55,267 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. 2024-11-20T05:15:55,268 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 63a460fa7bbfa3a642b55ff183f6f407: Waiting for close lock at 1732079753923Running coprocessor pre-close hooks at 1732079753923Disabling compacts and flushes for region at 1732079753923Disabling writes for close at 1732079753923Obtaining lock to block concurrent updates at 1732079753923Preparing flush snapshotting stores in 63a460fa7bbfa3a642b55ff183f6f407 at 1732079753923Finished memstore snapshotting TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732079753924 (+1 ms)Flushing stores of TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407. at 1732079753925 (+1 ms)Flushing 63a460fa7bbfa3a642b55ff183f6f407/A: creating writer at 1732079753925Flushing 63a460fa7bbfa3a642b55ff183f6f407/A: appending metadata at 1732079753927 (+2 ms)Flushing 63a460fa7bbfa3a642b55ff183f6f407/A: closing flushed file at 1732079753927Flushing 63a460fa7bbfa3a642b55ff183f6f407/B: creating writer at 1732079754336 (+409 ms)Flushing 63a460fa7bbfa3a642b55ff183f6f407/B: appending metadata at 1732079754340 (+4 ms)Flushing 63a460fa7bbfa3a642b55ff183f6f407/B: closing flushed file at 1732079754340Flushing 63a460fa7bbfa3a642b55ff183f6f407/C: creating writer at 1732079754753 (+413 ms)Flushing 63a460fa7bbfa3a642b55ff183f6f407/C: appending metadata at 1732079754756 (+3 ms)Flushing 63a460fa7bbfa3a642b55ff183f6f407/C: closing flushed file at 1732079754756Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29850c86: reopening flushed file at 1732079755164 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa6dbcb: reopening flushed file at 1732079755168 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27a84cca: reopening flushed file at 1732079755172 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 63a460fa7bbfa3a642b55ff183f6f407 in 1254ms, sequenceid=567, compaction requested=true at 1732079755177 (+5 ms)Writing region close event to WAL at 1732079755264 (+87 ms)Running coprocessor post-close hooks at 1732079755267 (+3 ms)Closed at 1732079755267 2024-11-20T05:15:55,269 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:55,270 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=63a460fa7bbfa3a642b55ff183f6f407, regionState=CLOSED 2024-11-20T05:15:55,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:15:55,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T05:15:55,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 63a460fa7bbfa3a642b55ff183f6f407, server=51d5d2011a1c,44753,1732079655401 in 1.5020 sec 2024-11-20T05:15:55,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-20T05:15:55,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63a460fa7bbfa3a642b55ff183f6f407, UNASSIGN in 1.5070 sec 2024-11-20T05:15:55,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T05:15:55,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-20T05:15:55,279 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079755279"}]},"ts":"1732079755279"} 2024-11-20T05:15:55,280 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T05:15:55,280 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T05:15:55,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5230 sec 2024-11-20T05:15:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T05:15:55,908 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:55,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T05:15:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:55,911 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T05:15:55,912 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:55,916 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:55,920 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/recovered.edits] 2024-11-20T05:15:55,924 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/31d6506ce74a4c20aa0db9e31fc99173 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/31d6506ce74a4c20aa0db9e31fc99173 2024-11-20T05:15:55,926 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5a36f7dace114a70a997a41d65fb9d67 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/5a36f7dace114a70a997a41d65fb9d67 2024-11-20T05:15:55,928 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/8f443fdc806e4745941c6906f8119a18 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/A/8f443fdc806e4745941c6906f8119a18 2024-11-20T05:15:55,931 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/07cb58fa73f3419f8d46793269766198 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/07cb58fa73f3419f8d46793269766198 2024-11-20T05:15:55,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4beba35a361f4708be5fce6dc6374f57 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/4beba35a361f4708be5fce6dc6374f57 2024-11-20T05:15:55,933 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/be3e5abaa1294a68a3419064fcf8b018 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/B/be3e5abaa1294a68a3419064fcf8b018 2024-11-20T05:15:55,936 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/28ef640d35a8422bb7c6c8772f83e6a4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/28ef640d35a8422bb7c6c8772f83e6a4 2024-11-20T05:15:55,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5d9d7c8e1fc48de8df851fe891593a9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/a5d9d7c8e1fc48de8df851fe891593a9 2024-11-20T05:15:55,938 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f5a0071fd7874213b84409bca87a367d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/C/f5a0071fd7874213b84409bca87a367d 2024-11-20T05:15:55,941 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/recovered.edits/570.seqid to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407/recovered.edits/570.seqid 2024-11-20T05:15:55,941 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/63a460fa7bbfa3a642b55ff183f6f407 2024-11-20T05:15:55,942 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T05:15:55,943 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:55,946 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T05:15:55,948 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T05:15:55,949 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:55,949 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T05:15:55,949 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732079755949"}]},"ts":"9223372036854775807"} 2024-11-20T05:15:55,951 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T05:15:55,951 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 63a460fa7bbfa3a642b55ff183f6f407, NAME => 'TestAcidGuarantees,,1732079729514.63a460fa7bbfa3a642b55ff183f6f407.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T05:15:55,951 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T05:15:55,951 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732079755951"}]},"ts":"9223372036854775807"} 2024-11-20T05:15:55,953 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T05:15:55,954 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:55,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-20T05:15:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T05:15:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T05:15:56,018 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:56,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_142706796_22 at /127.0.0.1:43500 [Waiting for operation #259] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-436410347_22 at /127.0.0.1:38376 [Waiting for operation #334] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 458) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5359 (was 4606) - AvailableMemoryMB LEAK? - 2024-11-20T05:15:56,041 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=5358 2024-11-20T05:15:56,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:15:56,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:15:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:56,046 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T05:15:56,046 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:56,046 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-20T05:15:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T05:15:56,047 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T05:15:56,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742133_1309 (size=1061) 2024-11-20T05:15:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T05:15:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T05:15:56,455 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:15:56,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742134_1310 (size=53) 2024-11-20T05:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T05:15:56,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:15:56,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e74402c4b2528f25097db0ff071d71d6, disabling compactions & flushes 2024-11-20T05:15:56,861 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:56,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:56,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. after waiting 0 ms 2024-11-20T05:15:56,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:56,861 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:56,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e74402c4b2528f25097db0ff071d71d6: Waiting for close lock at 1732079756861Disabling compacts and flushes for region at 1732079756861Disabling writes for close at 1732079756861Writing region close event to WAL at 1732079756861Closed at 1732079756861 2024-11-20T05:15:56,862 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T05:15:56,863 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732079756863"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732079756863"}]},"ts":"1732079756863"} 2024-11-20T05:15:56,865 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T05:15:56,865 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T05:15:56,866 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079756865"}]},"ts":"1732079756865"} 2024-11-20T05:15:56,867 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T05:15:56,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, ASSIGN}] 2024-11-20T05:15:56,868 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, ASSIGN 2024-11-20T05:15:56,869 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, ASSIGN; state=OFFLINE, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=false 2024-11-20T05:15:57,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:57,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, ASSIGN because future has completed 2024-11-20T05:15:57,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:15:57,177 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:57,177 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:15:57,177 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,177 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:15:57,177 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,177 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,179 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,180 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:57,180 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e74402c4b2528f25097db0ff071d71d6 columnFamilyName A 2024-11-20T05:15:57,180 DEBUG [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:57,181 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(327): Store=e74402c4b2528f25097db0ff071d71d6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:57,181 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,182 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:57,182 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e74402c4b2528f25097db0ff071d71d6 columnFamilyName B 2024-11-20T05:15:57,182 DEBUG [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:57,182 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(327): Store=e74402c4b2528f25097db0ff071d71d6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:57,182 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,183 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:57,183 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e74402c4b2528f25097db0ff071d71d6 columnFamilyName C 2024-11-20T05:15:57,183 DEBUG [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:57,184 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(327): Store=e74402c4b2528f25097db0ff071d71d6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:57,184 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:57,184 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,185 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,185 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,186 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,186 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,187 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T05:15:57,188 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,190 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:15:57,191 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened e74402c4b2528f25097db0ff071d71d6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59658707, jitterRate=-0.11101599037647247}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:15:57,191 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:57,192 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for e74402c4b2528f25097db0ff071d71d6: Running coprocessor pre-open hook at 1732079757177Writing region info on filesystem at 1732079757178 (+1 ms)Initializing all the Stores at 1732079757178Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079757178Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079757178Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079757179 (+1 ms)Cleaning up temporary data from old regions at 1732079757186 (+7 ms)Running coprocessor post-open hooks at 1732079757191 (+5 ms)Region opened successfully at 1732079757192 (+1 ms) 2024-11-20T05:15:57,193 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., pid=85, masterSystemTime=1732079757174 2024-11-20T05:15:57,195 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:57,195 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:57,195 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=OPEN, openSeqNum=2, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:57,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:15:57,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-20T05:15:57,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 in 177 msec 2024-11-20T05:15:57,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-20T05:15:57,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, ASSIGN in 334 msec 2024-11-20T05:15:57,205 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T05:15:57,206 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079757205"}]},"ts":"1732079757205"} 2024-11-20T05:15:57,208 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T05:15:57,209 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T05:15:57,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-20T05:15:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T05:15:58,198 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:58,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b84b66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:58,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:58,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:58,199 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a76c47e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:58,201 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569480d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:58,202 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:58,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:58,204 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T05:15:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 51d5d2011a1c,37927,1732079654666 2024-11-20T05:15:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4320992c 2024-11-20T05:15:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T05:15:58,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T05:15:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:15:58,207 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:15:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T05:15:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742135_1311 (size=1097) 2024-11-20T05:15:58,620 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T05:15:58,620 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T05:15:58,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:15:58,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, REOPEN/MOVE}] 2024-11-20T05:15:58,626 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, REOPEN/MOVE 2024-11-20T05:15:58,627 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:58,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, REOPEN/MOVE because future has completed 2024-11-20T05:15:58,630 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:15:58,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:15:58,782 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:58,782 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:15:58,783 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing e74402c4b2528f25097db0ff071d71d6, disabling compactions & flushes 2024-11-20T05:15:58,783 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:58,783 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:58,783 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. after waiting 0 ms 2024-11-20T05:15:58,783 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:58,787 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T05:15:58,788 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:58,788 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for e74402c4b2528f25097db0ff071d71d6: Waiting for close lock at 1732079758783Running coprocessor pre-close hooks at 1732079758783Disabling compacts and flushes for region at 1732079758783Disabling writes for close at 1732079758783Writing region close event to WAL at 1732079758783Running coprocessor post-close hooks at 1732079758787 (+4 ms)Closed at 1732079758788 (+1 ms) 2024-11-20T05:15:58,788 WARN [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: e74402c4b2528f25097db0ff071d71d6 to self. 2024-11-20T05:15:58,790 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:58,790 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=CLOSED 2024-11-20T05:15:58,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:15:58,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-20T05:15:58,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 in 163 msec 2024-11-20T05:15:58,797 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, REOPEN/MOVE; state=CLOSED, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=true 2024-11-20T05:15:58,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:58,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, REOPEN/MOVE because future has completed 2024-11-20T05:15:58,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:15:59,114 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,114 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:15:59,115 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,115 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:15:59,116 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,116 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,119 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,120 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:59,120 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e74402c4b2528f25097db0ff071d71d6 columnFamilyName A 2024-11-20T05:15:59,121 DEBUG [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:59,122 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(327): Store=e74402c4b2528f25097db0ff071d71d6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:59,122 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,123 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:59,123 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e74402c4b2528f25097db0ff071d71d6 columnFamilyName B 2024-11-20T05:15:59,123 DEBUG [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:59,123 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(327): Store=e74402c4b2528f25097db0ff071d71d6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:59,124 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,124 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:15:59,124 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e74402c4b2528f25097db0ff071d71d6 columnFamilyName C 2024-11-20T05:15:59,125 DEBUG [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:59,125 INFO [StoreOpener-e74402c4b2528f25097db0ff071d71d6-1 {}] regionserver.HStore(327): Store=e74402c4b2528f25097db0ff071d71d6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:15:59,125 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,125 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,126 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,127 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,128 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,128 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,129 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:15:59,133 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,134 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened e74402c4b2528f25097db0ff071d71d6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72054531, jitterRate=0.07369618117809296}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:15:59,134 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,136 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for e74402c4b2528f25097db0ff071d71d6: Running coprocessor pre-open hook at 1732079759116Writing region info on filesystem at 1732079759116Initializing all the Stores at 1732079759118 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079759118Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079759118Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079759119 (+1 ms)Cleaning up temporary data from old regions at 1732079759128 (+9 ms)Running coprocessor post-open hooks at 1732079759134 (+6 ms)Region opened successfully at 1732079759135 (+1 ms) 2024-11-20T05:15:59,143 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., pid=90, masterSystemTime=1732079759106 2024-11-20T05:15:59,145 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,145 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,146 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=OPEN, openSeqNum=5, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:15:59,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-20T05:15:59,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 in 197 msec 2024-11-20T05:15:59,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-20T05:15:59,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, REOPEN/MOVE in 526 msec 2024-11-20T05:15:59,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T05:15:59,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-20T05:15:59,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2024-11-20T05:15:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T05:15:59,159 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:15:59,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320fb02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,161 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d1bdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,163 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ce55ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79018c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,166 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c40e2fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5135584a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@725f7689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bc6c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,172 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9103de3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427fac46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5443d90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,175 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f048c78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a741c6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b71e4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5503582b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eab4bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,181 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d686acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,183 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746d0476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108eda1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3cb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,186 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14946b26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354663d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e61175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3083e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbfc9f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,197 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c421a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,199 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0600a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c87703e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:15:59,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:15:59,201 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:15:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:15:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:15:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b50c83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:15:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:15:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:15:59,203 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:15:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5337c328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:15:59,204 DEBUG [Thread-1447 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,204 DEBUG [Thread-1442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,204 DEBUG [Thread-1449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,209 DEBUG [Thread-1438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:15:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-20T05:15:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T05:15:59,216 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:15:59,217 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:15:59,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:15:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:15:59,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:15:59,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:59,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:15:59,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:59,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:15:59,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:15:59,223 DEBUG [Thread-1451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,226 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,231 DEBUG [Thread-1444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,235 DEBUG [Thread-1440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,236 DEBUG [Thread-1453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,252 DEBUG [Thread-1455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,255 DEBUG [Thread-1436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:15:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:15:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:15:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:15:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:15:59,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206450646ec6674af7bab794d0a4955c78_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079759220/Put/seqid=0 2024-11-20T05:15:59,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742136_1312 (size=14594) 2024-11-20T05:15:59,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079819264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079819273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079759256, max timestamp=1732079759277Num uniques 859; 2024-11-20T05:15:59,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:59,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079759256, max timestamp=1732079759277Num uniques 859; 2024-11-20T05:15:59,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079759256, max timestamp=1732079759277Num uniques 859; 2024-11-20T05:15:59,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:59,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:15:59,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079819314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T05:15:59,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079819316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079819317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:15:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:15:59,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079819387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079819387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079819427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079819438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079819438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:15:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:15:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T05:15:59,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079819597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079819597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079819647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079819647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079819647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,670 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:15:59,672 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206450646ec6674af7bab794d0a4955c78_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206450646ec6674af7bab794d0a4955c78_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:15:59,673 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/60446aa5adec4a5b9878d45ec4d7b273, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:15:59,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/60446aa5adec4a5b9878d45ec4d7b273 is 175, key is test_row_0/A:col10/1732079759220/Put/seqid=0 2024-11-20T05:15:59,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:15:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:15:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742137_1313 (size=39549) 2024-11-20T05:15:59,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:15:59,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:15:59,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T05:15:59,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079819917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079819917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079819957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079819957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:15:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079819957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:15:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:15:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:15:59,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:15:59,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:15:59,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:15:59,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:15:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/60446aa5adec4a5b9878d45ec4d7b273 2024-11-20T05:16:00,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/fe757a0ed3d94907ba7047832a5645cc is 50, key is test_row_0/B:col10/1732079759220/Put/seqid=0 2024-11-20T05:16:00,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742138_1314 (size=12001) 2024-11-20T05:16:00,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/fe757a0ed3d94907ba7047832a5645cc 2024-11-20T05:16:00,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5e597fafcc8c40e3b49460675e840102 is 50, key is test_row_0/C:col10/1732079759220/Put/seqid=0 2024-11-20T05:16:00,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742139_1315 (size=12001) 2024-11-20T05:16:00,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:16:00,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:00,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:00,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:00,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:16:00,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:00,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:00,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:00,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T05:16:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079820427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:00,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:16:00,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:00,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:00,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:00,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:00,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079820437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:00,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079820477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079820477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079820478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:00,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5e597fafcc8c40e3b49460675e840102 2024-11-20T05:16:00,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/60446aa5adec4a5b9878d45ec4d7b273 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/60446aa5adec4a5b9878d45ec4d7b273 2024-11-20T05:16:00,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/60446aa5adec4a5b9878d45ec4d7b273, entries=200, sequenceid=15, filesize=38.6 K 2024-11-20T05:16:00,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/fe757a0ed3d94907ba7047832a5645cc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/fe757a0ed3d94907ba7047832a5645cc 2024-11-20T05:16:00,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/fe757a0ed3d94907ba7047832a5645cc, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T05:16:00,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5e597fafcc8c40e3b49460675e840102 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5e597fafcc8c40e3b49460675e840102 2024-11-20T05:16:00,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5e597fafcc8c40e3b49460675e840102, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T05:16:00,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e74402c4b2528f25097db0ff071d71d6 in 1335ms, sequenceid=15, compaction requested=false 2024-11-20T05:16:00,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:00,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T05:16:00,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:00,594 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T05:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e3db1d4ade1a46a9a65752718e5d02c2_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079759316/Put/seqid=0 2024-11-20T05:16:00,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742140_1316 (size=12154) 2024-11-20T05:16:00,802 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T05:16:01,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:01,020 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e3db1d4ade1a46a9a65752718e5d02c2_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3db1d4ade1a46a9a65752718e5d02c2_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:01,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/6baa7a7086414c2bb2394b1fc1467b2c, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/6baa7a7086414c2bb2394b1fc1467b2c is 175, key is test_row_0/A:col10/1732079759316/Put/seqid=0 2024-11-20T05:16:01,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742141_1317 (size=30955) 2024-11-20T05:16:01,025 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/6baa7a7086414c2bb2394b1fc1467b2c 2024-11-20T05:16:01,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/1eeacb0f3d3645fab662f03f276d245c is 50, key is test_row_0/B:col10/1732079759316/Put/seqid=0 2024-11-20T05:16:01,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742142_1318 (size=12001) 2024-11-20T05:16:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T05:16:01,445 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/1eeacb0f3d3645fab662f03f276d245c 2024-11-20T05:16:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/af5df4a979dc4685b35eaf80992b1fe8 is 50, key is test_row_0/C:col10/1732079759316/Put/seqid=0 2024-11-20T05:16:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:01,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742143_1319 (size=12001) 2024-11-20T05:16:01,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079821494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079821498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,510 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079821502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079821508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079821508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,521 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,522 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079821607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079821628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079821827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079821848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:01,859 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/af5df4a979dc4685b35eaf80992b1fe8 2024-11-20T05:16:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/6baa7a7086414c2bb2394b1fc1467b2c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/6baa7a7086414c2bb2394b1fc1467b2c 2024-11-20T05:16:01,867 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/6baa7a7086414c2bb2394b1fc1467b2c, entries=150, sequenceid=50, filesize=30.2 K 2024-11-20T05:16:01,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/1eeacb0f3d3645fab662f03f276d245c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/1eeacb0f3d3645fab662f03f276d245c 2024-11-20T05:16:01,871 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/1eeacb0f3d3645fab662f03f276d245c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T05:16:01,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/af5df4a979dc4685b35eaf80992b1fe8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/af5df4a979dc4685b35eaf80992b1fe8 2024-11-20T05:16:01,875 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/af5df4a979dc4685b35eaf80992b1fe8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T05:16:01,876 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for e74402c4b2528f25097db0ff071d71d6 in 1283ms, sequenceid=50, compaction requested=false 2024-11-20T05:16:01,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:01,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:01,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-20T05:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-20T05:16:01,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-20T05:16:01,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6600 sec 2024-11-20T05:16:01,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.6640 sec 2024-11-20T05:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:02,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T05:16:02,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:02,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cfd22de0596748fa97c926886919ece1_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079762151/Put/seqid=0 2024-11-20T05:16:02,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742144_1320 (size=14594) 2024-11-20T05:16:02,173 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:02,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cfd22de0596748fa97c926886919ece1_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cfd22de0596748fa97c926886919ece1_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:02,177 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d79a8cb4fc1e42708ec98bf53319a9fc, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:02,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d79a8cb4fc1e42708ec98bf53319a9fc is 175, key is test_row_0/A:col10/1732079762151/Put/seqid=0 2024-11-20T05:16:02,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742145_1321 (size=39549) 2024-11-20T05:16:02,200 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d79a8cb4fc1e42708ec98bf53319a9fc 2024-11-20T05:16:02,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/471b349bc5cc4f5dbcac4e97923ce47b is 50, key is test_row_0/B:col10/1732079762151/Put/seqid=0 2024-11-20T05:16:02,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742146_1322 (size=12001) 2024-11-20T05:16:02,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/471b349bc5cc4f5dbcac4e97923ce47b 2024-11-20T05:16:02,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/ea600d02d5964189aa12d1c1ecf33952 is 50, key is test_row_0/C:col10/1732079762151/Put/seqid=0 2024-11-20T05:16:02,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742147_1323 (size=12001) 2024-11-20T05:16:02,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079822263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:02,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079822265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:02,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079822377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:02,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079822387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:02,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079822587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079822598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:02,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/ea600d02d5964189aa12d1c1ecf33952 2024-11-20T05:16:02,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d79a8cb4fc1e42708ec98bf53319a9fc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d79a8cb4fc1e42708ec98bf53319a9fc 2024-11-20T05:16:02,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d79a8cb4fc1e42708ec98bf53319a9fc, entries=200, sequenceid=62, filesize=38.6 K 2024-11-20T05:16:02,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/471b349bc5cc4f5dbcac4e97923ce47b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/471b349bc5cc4f5dbcac4e97923ce47b 2024-11-20T05:16:02,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/471b349bc5cc4f5dbcac4e97923ce47b, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T05:16:02,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/ea600d02d5964189aa12d1c1ecf33952 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ea600d02d5964189aa12d1c1ecf33952 2024-11-20T05:16:02,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ea600d02d5964189aa12d1c1ecf33952, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T05:16:02,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e74402c4b2528f25097db0ff071d71d6 in 499ms, sequenceid=62, compaction requested=true 2024-11-20T05:16:02,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:02,654 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:02,655 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:02,656 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/A is initiating minor compaction (all files) 2024-11-20T05:16:02,656 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/A in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:02,656 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/60446aa5adec4a5b9878d45ec4d7b273, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/6baa7a7086414c2bb2394b1fc1467b2c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d79a8cb4fc1e42708ec98bf53319a9fc] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=107.5 K 2024-11-20T05:16:02,656 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:02,656 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/60446aa5adec4a5b9878d45ec4d7b273, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/6baa7a7086414c2bb2394b1fc1467b2c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d79a8cb4fc1e42708ec98bf53319a9fc] 2024-11-20T05:16:02,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:02,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:02,657 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:02,657 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60446aa5adec4a5b9878d45ec4d7b273, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732079759218 2024-11-20T05:16:02,658 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6baa7a7086414c2bb2394b1fc1467b2c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732079759308 2024-11-20T05:16:02,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:02,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:02,658 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79a8cb4fc1e42708ec98bf53319a9fc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079761475 2024-11-20T05:16:02,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:02,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:02,658 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:02,658 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/B is initiating minor compaction (all files) 2024-11-20T05:16:02,658 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/B in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:02,658 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/fe757a0ed3d94907ba7047832a5645cc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/1eeacb0f3d3645fab662f03f276d245c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/471b349bc5cc4f5dbcac4e97923ce47b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.2 K 2024-11-20T05:16:02,659 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting fe757a0ed3d94907ba7047832a5645cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732079759218 2024-11-20T05:16:02,659 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eeacb0f3d3645fab662f03f276d245c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732079759308 2024-11-20T05:16:02,660 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 471b349bc5cc4f5dbcac4e97923ce47b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079761475 2024-11-20T05:16:02,664 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:02,669 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120763c7b096c814fb4a5992ae6f5906f8c_e74402c4b2528f25097db0ff071d71d6 store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:02,671 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120763c7b096c814fb4a5992ae6f5906f8c_e74402c4b2528f25097db0ff071d71d6, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:02,671 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120763c7b096c814fb4a5992ae6f5906f8c_e74402c4b2528f25097db0ff071d71d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:02,674 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:02,675 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/e18963ed21f74d6896880974a5dc079f is 50, key is test_row_0/B:col10/1732079762151/Put/seqid=0 2024-11-20T05:16:02,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742148_1324 (size=4469) 2024-11-20T05:16:02,691 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#A#compaction#275 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:02,692 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/32890ed9b31b4ed49e0502bd9fb73073 is 175, key is test_row_0/A:col10/1732079762151/Put/seqid=0 2024-11-20T05:16:02,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742149_1325 (size=12104) 2024-11-20T05:16:02,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742150_1326 (size=31058) 2024-11-20T05:16:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:02,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-20T05:16:02,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:02,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:02,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:02,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:02,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:02,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:02,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120adb5ed794e5f4298a7dbddfcd6041797_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079762912/Put/seqid=0 2024-11-20T05:16:02,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742151_1327 (size=14594) 2024-11-20T05:16:02,926 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:02,929 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120adb5ed794e5f4298a7dbddfcd6041797_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120adb5ed794e5f4298a7dbddfcd6041797_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:02,930 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/3ed06974f2e844ea86038d53e1139c49, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:02,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/3ed06974f2e844ea86038d53e1139c49 is 175, key is test_row_0/A:col10/1732079762912/Put/seqid=0 2024-11-20T05:16:02,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742152_1328 (size=39549) 2024-11-20T05:16:02,934 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/3ed06974f2e844ea86038d53e1139c49 2024-11-20T05:16:02,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/b2ad5260ef5046e689398a3eb3475e67 is 50, key is test_row_0/B:col10/1732079762912/Put/seqid=0 2024-11-20T05:16:02,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742153_1329 (size=12001) 2024-11-20T05:16:02,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079822949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:02,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079822956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079823067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079823067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,178 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/32890ed9b31b4ed49e0502bd9fb73073 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/32890ed9b31b4ed49e0502bd9fb73073 2024-11-20T05:16:03,179 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/e18963ed21f74d6896880974a5dc079f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/e18963ed21f74d6896880974a5dc079f 2024-11-20T05:16:03,185 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/B of e74402c4b2528f25097db0ff071d71d6 into e18963ed21f74d6896880974a5dc079f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:03,186 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:03,186 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/B, priority=13, startTime=1732079762657; duration=0sec 2024-11-20T05:16:03,186 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:03,186 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:B 2024-11-20T05:16:03,186 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:03,186 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/A of e74402c4b2528f25097db0ff071d71d6 into 32890ed9b31b4ed49e0502bd9fb73073(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:03,186 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:03,186 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/A, priority=13, startTime=1732079762654; duration=0sec 2024-11-20T05:16:03,186 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:03,186 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:A 2024-11-20T05:16:03,187 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:03,187 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/C is initiating minor compaction (all files) 2024-11-20T05:16:03,187 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/C in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:03,187 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5e597fafcc8c40e3b49460675e840102, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/af5df4a979dc4685b35eaf80992b1fe8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ea600d02d5964189aa12d1c1ecf33952] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.2 K 2024-11-20T05:16:03,188 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e597fafcc8c40e3b49460675e840102, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732079759218 2024-11-20T05:16:03,188 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting af5df4a979dc4685b35eaf80992b1fe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732079759308 2024-11-20T05:16:03,188 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ea600d02d5964189aa12d1c1ecf33952, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079761475 2024-11-20T05:16:03,196 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#C#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:03,196 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/3a56126202554ad2ae84b2b906dbd191 is 50, key is test_row_0/C:col10/1732079762151/Put/seqid=0 2024-11-20T05:16:03,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742154_1330 (size=12104) 2024-11-20T05:16:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079823277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079823287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/b2ad5260ef5046e689398a3eb3475e67 2024-11-20T05:16:03,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/38c789d3dcdc43a1826115baee1fed04 is 50, key is test_row_0/C:col10/1732079762912/Put/seqid=0 2024-11-20T05:16:03,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742155_1331 (size=12001) 2024-11-20T05:16:03,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/38c789d3dcdc43a1826115baee1fed04 2024-11-20T05:16:03,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/3ed06974f2e844ea86038d53e1139c49 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3ed06974f2e844ea86038d53e1139c49 2024-11-20T05:16:03,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3ed06974f2e844ea86038d53e1139c49, entries=200, sequenceid=83, filesize=38.6 K 2024-11-20T05:16:03,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/b2ad5260ef5046e689398a3eb3475e67 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/b2ad5260ef5046e689398a3eb3475e67 2024-11-20T05:16:03,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/b2ad5260ef5046e689398a3eb3475e67, entries=150, sequenceid=83, filesize=11.7 K 2024-11-20T05:16:03,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/38c789d3dcdc43a1826115baee1fed04 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/38c789d3dcdc43a1826115baee1fed04 2024-11-20T05:16:03,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/38c789d3dcdc43a1826115baee1fed04, entries=150, sequenceid=83, filesize=11.7 K 2024-11-20T05:16:03,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e74402c4b2528f25097db0ff071d71d6 in 463ms, sequenceid=83, compaction requested=false 2024-11-20T05:16:03,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T05:16:03,388 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:03,388 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-20T05:16:03,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T05:16:03,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:03,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T05:16:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:03,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T05:16:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:03,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:03,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fe45c53c8ef644bbaf40c41f4f5fc4be_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079762948/Put/seqid=0 2024-11-20T05:16:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742156_1332 (size=17034) 2024-11-20T05:16:03,607 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/3a56126202554ad2ae84b2b906dbd191 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/3a56126202554ad2ae84b2b906dbd191 2024-11-20T05:16:03,611 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/C of e74402c4b2528f25097db0ff071d71d6 into 3a56126202554ad2ae84b2b906dbd191(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:03,612 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:03,612 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/C, priority=13, startTime=1732079762658; duration=0sec 2024-11-20T05:16:03,612 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:03,612 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:C 2024-11-20T05:16:03,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079823607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079823617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079823617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079823617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079823618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:03,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T05:16:03,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079823737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079823737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079823738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079823738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:03,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:03,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:03,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:03,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:03,958 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:03,962 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fe45c53c8ef644bbaf40c41f4f5fc4be_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fe45c53c8ef644bbaf40c41f4f5fc4be_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:03,962 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/2ed482891c1d42dc9d229551d9b7478e, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:03,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/2ed482891c1d42dc9d229551d9b7478e is 175, key is test_row_0/A:col10/1732079762948/Put/seqid=0 2024-11-20T05:16:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742157_1333 (size=48139) 2024-11-20T05:16:03,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079823957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079823957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079823957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079823957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:04,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:04,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T05:16:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079824127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079824278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079824278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079824278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079824278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,366 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/2ed482891c1d42dc9d229551d9b7478e 2024-11-20T05:16:04,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/9b51e461928647e9acd23e9a5972c6a6 is 50, key is test_row_0/B:col10/1732079762948/Put/seqid=0 2024-11-20T05:16:04,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742158_1334 (size=12001) 2024-11-20T05:16:04,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:04,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T05:16:04,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:04,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:04,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:04,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:04,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/9b51e461928647e9acd23e9a5972c6a6 2024-11-20T05:16:04,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079824799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079824799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079824799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079824799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:04,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/7f2e7839aac4402c8c1d28cfe2ee40c6 is 50, key is test_row_0/C:col10/1732079762948/Put/seqid=0 2024-11-20T05:16:04,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742159_1335 (size=12001) 2024-11-20T05:16:04,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:04,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:04,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:04,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:05,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:05,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:05,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:05,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079825153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,161 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:05,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:05,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:05,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:05,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/7f2e7839aac4402c8c1d28cfe2ee40c6 2024-11-20T05:16:05,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/2ed482891c1d42dc9d229551d9b7478e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/2ed482891c1d42dc9d229551d9b7478e 2024-11-20T05:16:05,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/2ed482891c1d42dc9d229551d9b7478e, entries=250, sequenceid=98, filesize=47.0 K 2024-11-20T05:16:05,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/9b51e461928647e9acd23e9a5972c6a6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/9b51e461928647e9acd23e9a5972c6a6 2024-11-20T05:16:05,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/9b51e461928647e9acd23e9a5972c6a6, entries=150, sequenceid=98, filesize=11.7 K 2024-11-20T05:16:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/7f2e7839aac4402c8c1d28cfe2ee40c6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/7f2e7839aac4402c8c1d28cfe2ee40c6 2024-11-20T05:16:05,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/7f2e7839aac4402c8c1d28cfe2ee40c6, entries=150, sequenceid=98, filesize=11.7 K 2024-11-20T05:16:05,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e74402c4b2528f25097db0ff071d71d6 in 1750ms, sequenceid=98, compaction requested=true 2024-11-20T05:16:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:05,282 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:05,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T05:16:05,282 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:05,283 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:05,283 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:05,283 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/B is initiating minor compaction (all files) 2024-11-20T05:16:05,283 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/A is initiating minor compaction (all files) 2024-11-20T05:16:05,283 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/B in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,283 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/A in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,284 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/e18963ed21f74d6896880974a5dc079f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/b2ad5260ef5046e689398a3eb3475e67, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/9b51e461928647e9acd23e9a5972c6a6] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.3 K 2024-11-20T05:16:05,284 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/32890ed9b31b4ed49e0502bd9fb73073, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3ed06974f2e844ea86038d53e1139c49, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/2ed482891c1d42dc9d229551d9b7478e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=116.0 K 2024-11-20T05:16:05,284 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,284 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/32890ed9b31b4ed49e0502bd9fb73073, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3ed06974f2e844ea86038d53e1139c49, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/2ed482891c1d42dc9d229551d9b7478e] 2024-11-20T05:16:05,284 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e18963ed21f74d6896880974a5dc079f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079761475 2024-11-20T05:16:05,287 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 32890ed9b31b4ed49e0502bd9fb73073, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079761475 2024-11-20T05:16:05,287 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ad5260ef5046e689398a3eb3475e67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732079762262 2024-11-20T05:16:05,289 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed06974f2e844ea86038d53e1139c49, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732079762249 2024-11-20T05:16:05,290 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b51e461928647e9acd23e9a5972c6a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732079762941 2024-11-20T05:16:05,290 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ed482891c1d42dc9d229551d9b7478e, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732079762941 2024-11-20T05:16:05,305 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:05,322 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:05,323 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d3c2aad1defa4fbbbbb068dd77e42267 is 50, key is test_row_0/B:col10/1732079762948/Put/seqid=0 2024-11-20T05:16:05,345 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b93df25496c04c7d969ee205fc031daa_e74402c4b2528f25097db0ff071d71d6 store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:05,347 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b93df25496c04c7d969ee205fc031daa_e74402c4b2528f25097db0ff071d71d6, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:05,347 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b93df25496c04c7d969ee205fc031daa_e74402c4b2528f25097db0ff071d71d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742161_1337 (size=4469) 2024-11-20T05:16:05,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T05:16:05,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742160_1336 (size=12207) 2024-11-20T05:16:05,399 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#A#compaction#284 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:05,399 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T05:16:05,399 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/80ee3a5ac807472c96354d7c9d2e78ec is 175, key is test_row_0/A:col10/1732079762948/Put/seqid=0 2024-11-20T05:16:05,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:05,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:05,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:05,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:05,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:05,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:05,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112044395dd07576480e8bdb430b7973a103_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079763616/Put/seqid=0 2024-11-20T05:16:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742162_1338 (size=31161) 2024-11-20T05:16:05,466 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/80ee3a5ac807472c96354d7c9d2e78ec as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/80ee3a5ac807472c96354d7c9d2e78ec 2024-11-20T05:16:05,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742163_1339 (size=12154) 2024-11-20T05:16:05,479 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/A of e74402c4b2528f25097db0ff071d71d6 into 80ee3a5ac807472c96354d7c9d2e78ec(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:05,479 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:05,479 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/A, priority=13, startTime=1732079765282; duration=0sec 2024-11-20T05:16:05,480 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:05,480 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:A 2024-11-20T05:16:05,480 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:05,481 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:05,481 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/C is initiating minor compaction (all files) 2024-11-20T05:16:05,481 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/C in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:05,481 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/3a56126202554ad2ae84b2b906dbd191, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/38c789d3dcdc43a1826115baee1fed04, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/7f2e7839aac4402c8c1d28cfe2ee40c6] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.3 K 2024-11-20T05:16:05,482 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a56126202554ad2ae84b2b906dbd191, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079761475 2024-11-20T05:16:05,482 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c789d3dcdc43a1826115baee1fed04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732079762262 2024-11-20T05:16:05,483 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2e7839aac4402c8c1d28cfe2ee40c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732079762941 2024-11-20T05:16:05,511 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:05,511 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/111ccb13c6dc404588c33df1d2f9831a is 50, key is test_row_0/C:col10/1732079762948/Put/seqid=0 2024-11-20T05:16:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T05:16:05,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742164_1340 (size=12207) 2024-11-20T05:16:05,548 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/111ccb13c6dc404588c33df1d2f9831a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/111ccb13c6dc404588c33df1d2f9831a 2024-11-20T05:16:05,557 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/C of e74402c4b2528f25097db0ff071d71d6 into 111ccb13c6dc404588c33df1d2f9831a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:05,557 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:05,557 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/C, priority=13, startTime=1732079765282; duration=0sec 2024-11-20T05:16:05,557 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:05,557 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:C 2024-11-20T05:16:05,804 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d3c2aad1defa4fbbbbb068dd77e42267 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d3c2aad1defa4fbbbbb068dd77e42267 2024-11-20T05:16:05,808 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/B of e74402c4b2528f25097db0ff071d71d6 into d3c2aad1defa4fbbbbb068dd77e42267(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:05,808 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:05,808 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/B, priority=13, startTime=1732079765282; duration=0sec 2024-11-20T05:16:05,808 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:05,808 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:B 2024-11-20T05:16:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:05,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:05,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079825844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079825844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079825851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079825851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:05,876 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112044395dd07576480e8bdb430b7973a103_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112044395dd07576480e8bdb430b7973a103_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:05,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d2c1300e62114935a5549aee638cdb52, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:05,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d2c1300e62114935a5549aee638cdb52 is 175, key is test_row_0/A:col10/1732079763616/Put/seqid=0 2024-11-20T05:16:05,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742165_1341 (size=30955) 2024-11-20T05:16:05,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079825957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079825957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079825967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079825967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079826167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079826167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079826187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079826187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,287 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d2c1300e62114935a5549aee638cdb52 2024-11-20T05:16:06,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d8390c42311845d0bc189a9576029b3b is 50, key is test_row_0/B:col10/1732079763616/Put/seqid=0 2024-11-20T05:16:06,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742166_1342 (size=12001) 2024-11-20T05:16:06,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079826477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079826487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079826507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079826507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:06,698 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d8390c42311845d0bc189a9576029b3b 2024-11-20T05:16:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/be4a680d9ae54c9b9e02894ade8413fe is 50, key is test_row_0/C:col10/1732079763616/Put/seqid=0 2024-11-20T05:16:06,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742167_1343 (size=12001) 2024-11-20T05:16:07,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079826997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:07,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079827007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079827027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079827027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:07,122 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/be4a680d9ae54c9b9e02894ade8413fe 2024-11-20T05:16:07,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/d2c1300e62114935a5549aee638cdb52 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d2c1300e62114935a5549aee638cdb52 2024-11-20T05:16:07,130 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d2c1300e62114935a5549aee638cdb52, entries=150, sequenceid=118, filesize=30.2 K 2024-11-20T05:16:07,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d8390c42311845d0bc189a9576029b3b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d8390c42311845d0bc189a9576029b3b 2024-11-20T05:16:07,137 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d8390c42311845d0bc189a9576029b3b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-20T05:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/be4a680d9ae54c9b9e02894ade8413fe as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/be4a680d9ae54c9b9e02894ade8413fe 2024-11-20T05:16:07,143 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/be4a680d9ae54c9b9e02894ade8413fe, entries=150, sequenceid=118, filesize=11.7 K 2024-11-20T05:16:07,144 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e74402c4b2528f25097db0ff071d71d6 in 1745ms, sequenceid=118, compaction requested=false 2024-11-20T05:16:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-20T05:16:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-20T05:16:07,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-20T05:16:07,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7530 sec 2024-11-20T05:16:07,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.7590 sec 2024-11-20T05:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:07,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T05:16:07,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:07,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:07,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:07,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:07,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:07,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:07,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112054811b19297c4b38909e6ec06bf1ec38_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079765850/Put/seqid=0 2024-11-20T05:16:07,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742168_1344 (size=14794) 2024-11-20T05:16:07,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079827306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079827417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T05:16:07,567 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:07,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-20T05:16:07,571 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:07,572 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:07,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T05:16:07,608 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:07,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112054811b19297c4b38909e6ec06bf1ec38_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112054811b19297c4b38909e6ec06bf1ec38_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:07,612 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/f39763ed369e4868b86e734de9a730a8, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:07,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/f39763ed369e4868b86e734de9a730a8 is 175, key is test_row_0/A:col10/1732079765850/Put/seqid=0 2024-11-20T05:16:07,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742169_1345 (size=39749) 2024-11-20T05:16:07,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/f39763ed369e4868b86e734de9a730a8 2024-11-20T05:16:07,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/7e62ecf3df1e4c608b65c6d9ca94a90e is 50, key is test_row_0/B:col10/1732079765850/Put/seqid=0 2024-11-20T05:16:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742170_1346 (size=12151) 2024-11-20T05:16:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079827637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T05:16:07,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T05:16:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:07,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:07,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T05:16:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:07,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T05:16:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079827957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:08,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079828017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:08,027 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:08,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079828027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:08,031 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:08,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/7e62ecf3df1e4c608b65c6d9ca94a90e 2024-11-20T05:16:08,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/c03665d8bc184087b24b779a2678e79c is 50, key is test_row_0/C:col10/1732079765850/Put/seqid=0 2024-11-20T05:16:08,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T05:16:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:08,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:08,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742171_1347 (size=12151) 2024-11-20T05:16:08,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/c03665d8bc184087b24b779a2678e79c 2024-11-20T05:16:08,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079828047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079828047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:08,057 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:08,059 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:08,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/f39763ed369e4868b86e734de9a730a8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/f39763ed369e4868b86e734de9a730a8 2024-11-20T05:16:08,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/f39763ed369e4868b86e734de9a730a8, entries=200, sequenceid=136, filesize=38.8 K 2024-11-20T05:16:08,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/7e62ecf3df1e4c608b65c6d9ca94a90e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7e62ecf3df1e4c608b65c6d9ca94a90e 2024-11-20T05:16:08,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7e62ecf3df1e4c608b65c6d9ca94a90e, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T05:16:08,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/c03665d8bc184087b24b779a2678e79c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c03665d8bc184087b24b779a2678e79c 2024-11-20T05:16:08,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c03665d8bc184087b24b779a2678e79c, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T05:16:08,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e74402c4b2528f25097db0ff071d71d6 in 882ms, sequenceid=136, compaction requested=true 2024-11-20T05:16:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:08,077 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T05:16:08,078 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:08,078 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/A is initiating minor compaction (all files) 2024-11-20T05:16:08,078 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/A in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:08,078 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/80ee3a5ac807472c96354d7c9d2e78ec, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d2c1300e62114935a5549aee638cdb52, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/f39763ed369e4868b86e734de9a730a8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=99.5 K 2024-11-20T05:16:08,078 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:08,079 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/80ee3a5ac807472c96354d7c9d2e78ec, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d2c1300e62114935a5549aee638cdb52, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/f39763ed369e4868b86e734de9a730a8] 2024-11-20T05:16:08,079 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:08,079 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ee3a5ac807472c96354d7c9d2e78ec, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732079762941 2024-11-20T05:16:08,079 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2c1300e62114935a5549aee638cdb52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732079763601 2024-11-20T05:16:08,079 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:08,079 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f39763ed369e4868b86e734de9a730a8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732079765843 2024-11-20T05:16:08,079 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/B is initiating minor compaction (all files) 2024-11-20T05:16:08,080 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/B in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:08,080 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d3c2aad1defa4fbbbbb068dd77e42267, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d8390c42311845d0bc189a9576029b3b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7e62ecf3df1e4c608b65c6d9ca94a90e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.5 K 2024-11-20T05:16:08,080 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c2aad1defa4fbbbbb068dd77e42267, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732079762941 2024-11-20T05:16:08,080 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d8390c42311845d0bc189a9576029b3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732079763601 2024-11-20T05:16:08,081 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e62ecf3df1e4c608b65c6d9ca94a90e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732079765843 2024-11-20T05:16:08,097 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:08,099 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:08,100 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/4969496eb9ed457f94f074220b5d9166 is 50, key is test_row_0/B:col10/1732079765850/Put/seqid=0 2024-11-20T05:16:08,105 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411205439c71a984f4a32963a1eebc63af459_e74402c4b2528f25097db0ff071d71d6 store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:08,107 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411205439c71a984f4a32963a1eebc63af459_e74402c4b2528f25097db0ff071d71d6, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:08,107 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205439c71a984f4a32963a1eebc63af459_e74402c4b2528f25097db0ff071d71d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:08,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742172_1348 (size=12459) 2024-11-20T05:16:08,119 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/4969496eb9ed457f94f074220b5d9166 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/4969496eb9ed457f94f074220b5d9166 2024-11-20T05:16:08,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742173_1349 (size=4469) 2024-11-20T05:16:08,124 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/B of e74402c4b2528f25097db0ff071d71d6 into 4969496eb9ed457f94f074220b5d9166(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:08,124 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:08,124 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/B, priority=13, startTime=1732079768077; duration=0sec 2024-11-20T05:16:08,124 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:08,124 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:B 2024-11-20T05:16:08,125 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:08,125 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#A#compaction#293 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:08,126 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:08,126 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/1b0bd16f75794696a614151bd5ab5127 is 175, key is test_row_0/A:col10/1732079765850/Put/seqid=0 2024-11-20T05:16:08,126 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/C is initiating minor compaction (all files) 2024-11-20T05:16:08,126 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/C in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:08,126 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/111ccb13c6dc404588c33df1d2f9831a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/be4a680d9ae54c9b9e02894ade8413fe, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c03665d8bc184087b24b779a2678e79c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.5 K 2024-11-20T05:16:08,126 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 111ccb13c6dc404588c33df1d2f9831a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732079762941 2024-11-20T05:16:08,126 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting be4a680d9ae54c9b9e02894ade8413fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732079763601 2024-11-20T05:16:08,127 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c03665d8bc184087b24b779a2678e79c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732079765843 2024-11-20T05:16:08,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742174_1350 (size=31413) 2024-11-20T05:16:08,188 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#C#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:08,188 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/96c7591bb5224c71ad197aa2c5076d4b is 50, key is test_row_0/C:col10/1732079765850/Put/seqid=0 2024-11-20T05:16:08,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T05:16:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:08,196 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:16:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:08,197 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/1b0bd16f75794696a614151bd5ab5127 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/1b0bd16f75794696a614151bd5ab5127 2024-11-20T05:16:08,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742175_1351 (size=12459) 2024-11-20T05:16:08,204 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/A of e74402c4b2528f25097db0ff071d71d6 into 1b0bd16f75794696a614151bd5ab5127(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:08,204 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:08,204 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/A, priority=13, startTime=1732079768077; duration=0sec 2024-11-20T05:16:08,204 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:08,204 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:A 2024-11-20T05:16:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cdbc69521dcf47b3a1f4ff7605f42f4c_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079767305/Put/seqid=0 2024-11-20T05:16:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T05:16:08,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742176_1352 (size=12304) 2024-11-20T05:16:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:08,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:08,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079828539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:08,602 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/96c7591bb5224c71ad197aa2c5076d4b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/96c7591bb5224c71ad197aa2c5076d4b 2024-11-20T05:16:08,606 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/C of e74402c4b2528f25097db0ff071d71d6 into 96c7591bb5224c71ad197aa2c5076d4b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:08,606 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:08,606 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/C, priority=13, startTime=1732079768077; duration=0sec 2024-11-20T05:16:08,606 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:08,606 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:C 2024-11-20T05:16:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:08,629 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cdbc69521dcf47b3a1f4ff7605f42f4c_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdbc69521dcf47b3a1f4ff7605f42f4c_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fff1b375eaaf483ca97aed138c3e109a, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fff1b375eaaf483ca97aed138c3e109a is 175, key is test_row_0/A:col10/1732079767305/Put/seqid=0 2024-11-20T05:16:08,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742177_1353 (size=31105) 2024-11-20T05:16:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079828647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T05:16:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079828867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:09,036 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fff1b375eaaf483ca97aed138c3e109a 2024-11-20T05:16:09,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d2c5bd62eaef4059a1c4ead3b7e79ac5 is 50, key is test_row_0/B:col10/1732079767305/Put/seqid=0 2024-11-20T05:16:09,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742178_1354 (size=12151) 2024-11-20T05:16:09,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079829187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:09,447 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d2c5bd62eaef4059a1c4ead3b7e79ac5 2024-11-20T05:16:09,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/50cf9032e48c46e585c0da0a92e3fe8b is 50, key is test_row_0/C:col10/1732079767305/Put/seqid=0 2024-11-20T05:16:09,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742179_1355 (size=12151) 2024-11-20T05:16:09,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079829697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T05:16:09,857 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/50cf9032e48c46e585c0da0a92e3fe8b 2024-11-20T05:16:09,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fff1b375eaaf483ca97aed138c3e109a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fff1b375eaaf483ca97aed138c3e109a 2024-11-20T05:16:09,866 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fff1b375eaaf483ca97aed138c3e109a, entries=150, sequenceid=154, filesize=30.4 K 2024-11-20T05:16:09,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/d2c5bd62eaef4059a1c4ead3b7e79ac5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d2c5bd62eaef4059a1c4ead3b7e79ac5 2024-11-20T05:16:09,884 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d2c5bd62eaef4059a1c4ead3b7e79ac5, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T05:16:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/50cf9032e48c46e585c0da0a92e3fe8b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/50cf9032e48c46e585c0da0a92e3fe8b 2024-11-20T05:16:09,890 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/50cf9032e48c46e585c0da0a92e3fe8b, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T05:16:09,892 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e74402c4b2528f25097db0ff071d71d6 in 1696ms, sequenceid=154, compaction requested=false 2024-11-20T05:16:09,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:09,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:09,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-20T05:16:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-20T05:16:09,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-20T05:16:09,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-11-20T05:16:09,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.3280 sec 2024-11-20T05:16:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:10,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:16:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:10,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f0edac34261b48abb7f03546fdd08d59_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079768528/Put/seqid=0 2024-11-20T05:16:10,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742180_1356 (size=14794) 2024-11-20T05:16:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079830086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079830088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079830088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079830089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079830197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079830207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079830207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079830207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079830417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079830417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079830417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079830417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:10,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f0edac34261b48abb7f03546fdd08d59_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0edac34261b48abb7f03546fdd08d59_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:10,454 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5830240dd23f44c58bcb31540917a3b4, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:10,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5830240dd23f44c58bcb31540917a3b4 is 175, key is test_row_0/A:col10/1732079768528/Put/seqid=0 2024-11-20T05:16:10,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742181_1357 (size=39749) 2024-11-20T05:16:10,459 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5830240dd23f44c58bcb31540917a3b4 2024-11-20T05:16:10,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/c3c9dc8131f54e1780ed3a71d6b219c0 is 50, key is test_row_0/B:col10/1732079768528/Put/seqid=0 2024-11-20T05:16:10,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742182_1358 (size=12151) 2024-11-20T05:16:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079830707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,714 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079830727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079830727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079830727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079830727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:10,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/c3c9dc8131f54e1780ed3a71d6b219c0 2024-11-20T05:16:10,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5aed998b0a2448a3b23f448d43e7425b is 50, key is test_row_0/C:col10/1732079768528/Put/seqid=0 2024-11-20T05:16:10,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742183_1359 (size=12151) 2024-11-20T05:16:11,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079831248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:11,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079831248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079831249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:11,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079831249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:11,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5aed998b0a2448a3b23f448d43e7425b 2024-11-20T05:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5830240dd23f44c58bcb31540917a3b4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5830240dd23f44c58bcb31540917a3b4 2024-11-20T05:16:11,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5830240dd23f44c58bcb31540917a3b4, entries=200, sequenceid=172, filesize=38.8 K 2024-11-20T05:16:11,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/c3c9dc8131f54e1780ed3a71d6b219c0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c3c9dc8131f54e1780ed3a71d6b219c0 2024-11-20T05:16:11,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c3c9dc8131f54e1780ed3a71d6b219c0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-20T05:16:11,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5aed998b0a2448a3b23f448d43e7425b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5aed998b0a2448a3b23f448d43e7425b 2024-11-20T05:16:11,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5aed998b0a2448a3b23f448d43e7425b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-20T05:16:11,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e74402c4b2528f25097db0ff071d71d6 in 1278ms, sequenceid=172, compaction requested=true 2024-11-20T05:16:11,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:11,319 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:11,320 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:11,320 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/A is initiating minor compaction (all files) 2024-11-20T05:16:11,320 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/A in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:11,320 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/1b0bd16f75794696a614151bd5ab5127, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fff1b375eaaf483ca97aed138c3e109a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5830240dd23f44c58bcb31540917a3b4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=99.9 K 2024-11-20T05:16:11,320 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:11,321 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/1b0bd16f75794696a614151bd5ab5127, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fff1b375eaaf483ca97aed138c3e109a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5830240dd23f44c58bcb31540917a3b4] 2024-11-20T05:16:11,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:11,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:11,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:11,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:11,321 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:11,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:11,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:11,321 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0bd16f75794696a614151bd5ab5127, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732079765843 2024-11-20T05:16:11,321 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fff1b375eaaf483ca97aed138c3e109a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732079767265 2024-11-20T05:16:11,321 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5830240dd23f44c58bcb31540917a3b4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732079768510 2024-11-20T05:16:11,324 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:11,324 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/B is initiating minor compaction (all files) 2024-11-20T05:16:11,324 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/B in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:11,324 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/4969496eb9ed457f94f074220b5d9166, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d2c5bd62eaef4059a1c4ead3b7e79ac5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c3c9dc8131f54e1780ed3a71d6b219c0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.9 K 2024-11-20T05:16:11,325 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4969496eb9ed457f94f074220b5d9166, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732079765843 2024-11-20T05:16:11,325 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c5bd62eaef4059a1c4ead3b7e79ac5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732079767265 2024-11-20T05:16:11,325 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c3c9dc8131f54e1780ed3a71d6b219c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732079768528 2024-11-20T05:16:11,333 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:11,336 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:11,337 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/79783fd16c0f448eaf28eee246f9d46e is 50, key is test_row_0/B:col10/1732079768528/Put/seqid=0 2024-11-20T05:16:11,341 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112024da9f41f0ca4b9a9b00320e726aeedf_e74402c4b2528f25097db0ff071d71d6 store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:11,342 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112024da9f41f0ca4b9a9b00320e726aeedf_e74402c4b2528f25097db0ff071d71d6, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:11,343 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112024da9f41f0ca4b9a9b00320e726aeedf_e74402c4b2528f25097db0ff071d71d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:11,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742184_1360 (size=12561) 2024-11-20T05:16:11,354 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/79783fd16c0f448eaf28eee246f9d46e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/79783fd16c0f448eaf28eee246f9d46e 2024-11-20T05:16:11,362 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/B of e74402c4b2528f25097db0ff071d71d6 into 79783fd16c0f448eaf28eee246f9d46e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:11,362 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:11,362 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/B, priority=13, startTime=1732079771321; duration=0sec 2024-11-20T05:16:11,362 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:11,362 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:B 2024-11-20T05:16:11,362 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:11,364 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:11,364 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/C is initiating minor compaction (all files) 2024-11-20T05:16:11,364 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/C in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:11,364 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/96c7591bb5224c71ad197aa2c5076d4b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/50cf9032e48c46e585c0da0a92e3fe8b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5aed998b0a2448a3b23f448d43e7425b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=35.9 K 2024-11-20T05:16:11,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742185_1361 (size=4469) 2024-11-20T05:16:11,365 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c7591bb5224c71ad197aa2c5076d4b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732079765843 2024-11-20T05:16:11,365 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 50cf9032e48c46e585c0da0a92e3fe8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732079767265 2024-11-20T05:16:11,365 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aed998b0a2448a3b23f448d43e7425b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732079768528 2024-11-20T05:16:11,374 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:11,375 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/0f618004dd574d0bba130fd72ebe7bf6 is 50, key is test_row_0/C:col10/1732079768528/Put/seqid=0 2024-11-20T05:16:11,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742186_1362 (size=12561) 2024-11-20T05:16:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T05:16:11,747 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:11,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-20T05:16:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T05:16:11,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:11,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:11,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:11,755 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#A#compaction#302 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:11,756 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/a36abf1d36bd40dc9e0919af94bf8da9 is 175, key is test_row_0/A:col10/1732079768528/Put/seqid=0 2024-11-20T05:16:11,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742187_1363 (size=31515) 2024-11-20T05:16:11,793 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/0f618004dd574d0bba130fd72ebe7bf6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0f618004dd574d0bba130fd72ebe7bf6 2024-11-20T05:16:11,797 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/C of e74402c4b2528f25097db0ff071d71d6 into 0f618004dd574d0bba130fd72ebe7bf6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:11,797 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:11,797 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/C, priority=13, startTime=1732079771321; duration=0sec 2024-11-20T05:16:11,798 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:11,798 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:C 2024-11-20T05:16:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T05:16:11,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T05:16:11,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:11,902 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:16:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:11,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f987add547914ff8bc17dd715ae174d1_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079770085/Put/seqid=0 2024-11-20T05:16:11,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742188_1364 (size=12304) 2024-11-20T05:16:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T05:16:12,163 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/a36abf1d36bd40dc9e0919af94bf8da9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/a36abf1d36bd40dc9e0919af94bf8da9 2024-11-20T05:16:12,167 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/A of e74402c4b2528f25097db0ff071d71d6 into a36abf1d36bd40dc9e0919af94bf8da9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:12,167 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:12,167 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/A, priority=13, startTime=1732079771319; duration=0sec 2024-11-20T05:16:12,167 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:12,167 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:A 2024-11-20T05:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079832287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079832294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079832295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079832300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:12,316 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f987add547914ff8bc17dd715ae174d1_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f987add547914ff8bc17dd715ae174d1_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:12,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5b8694ae0e034ebeb7d24100d17e0a53, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:12,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5b8694ae0e034ebeb7d24100d17e0a53 is 175, key is test_row_0/A:col10/1732079770085/Put/seqid=0 2024-11-20T05:16:12,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742189_1365 (size=31105) 2024-11-20T05:16:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T05:16:12,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079832407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079832407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079832407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079832407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079832627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079832627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079832627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079832627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,722 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5b8694ae0e034ebeb7d24100d17e0a53 2024-11-20T05:16:12,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/59616e60d52540eb99a640a0061c8787 is 50, key is test_row_0/B:col10/1732079770085/Put/seqid=0 2024-11-20T05:16:12,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079832737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,749 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742190_1366 (size=12151) 2024-11-20T05:16:12,765 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/59616e60d52540eb99a640a0061c8787 2024-11-20T05:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/ce1eaedc610844428e83835930e5b1cd is 50, key is test_row_0/C:col10/1732079770085/Put/seqid=0 2024-11-20T05:16:12,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742191_1367 (size=12151) 2024-11-20T05:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T05:16:12,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079832937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079832938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079832938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079832938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,235 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/ce1eaedc610844428e83835930e5b1cd 2024-11-20T05:16:13,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/5b8694ae0e034ebeb7d24100d17e0a53 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5b8694ae0e034ebeb7d24100d17e0a53 2024-11-20T05:16:13,257 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5b8694ae0e034ebeb7d24100d17e0a53, entries=150, sequenceid=190, filesize=30.4 K 2024-11-20T05:16:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/59616e60d52540eb99a640a0061c8787 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/59616e60d52540eb99a640a0061c8787 2024-11-20T05:16:13,268 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/59616e60d52540eb99a640a0061c8787, entries=150, sequenceid=190, filesize=11.9 K 2024-11-20T05:16:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/ce1eaedc610844428e83835930e5b1cd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ce1eaedc610844428e83835930e5b1cd 2024-11-20T05:16:13,273 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ce1eaedc610844428e83835930e5b1cd, entries=150, sequenceid=190, filesize=11.9 K 2024-11-20T05:16:13,274 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e74402c4b2528f25097db0ff071d71d6 in 1372ms, sequenceid=190, compaction requested=false 2024-11-20T05:16:13,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:13,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:13,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-20T05:16:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-20T05:16:13,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-20T05:16:13,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5260 sec 2024-11-20T05:16:13,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5310 sec 2024-11-20T05:16:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:13,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:16:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079833486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079833492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200386dc9d6b0e4234bf58065bb66eecdc_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079772298/Put/seqid=0 2024-11-20T05:16:13,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079833499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079833501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742192_1368 (size=14794) 2024-11-20T05:16:13,533 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:13,543 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200386dc9d6b0e4234bf58065bb66eecdc_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200386dc9d6b0e4234bf58065bb66eecdc_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:13,544 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fbac9d214c4d4a5f8f4a76216d466834, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:13,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fbac9d214c4d4a5f8f4a76216d466834 is 175, key is test_row_0/A:col10/1732079772298/Put/seqid=0 2024-11-20T05:16:13,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742193_1369 (size=39749) 2024-11-20T05:16:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079833607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079833607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079833627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079833628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,790 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T05:16:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079833827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079833827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079833847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079833847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T05:16:13,898 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:13,898 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-20T05:16:13,900 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T05:16:13,900 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:13,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:13,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fbac9d214c4d4a5f8f4a76216d466834 2024-11-20T05:16:13,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/712d58357e094b24bc1a77ce109a1c1e is 50, key is test_row_0/B:col10/1732079772298/Put/seqid=0 2024-11-20T05:16:13,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742194_1370 (size=12151) 2024-11-20T05:16:13,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/712d58357e094b24bc1a77ce109a1c1e 2024-11-20T05:16:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T05:16:14,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/c8030b45318245bdbeba1e87e0134d67 is 50, key is test_row_0/C:col10/1732079772298/Put/seqid=0 2024-11-20T05:16:14,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742195_1371 (size=12151) 2024-11-20T05:16:14,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T05:16:14,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:14,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079834147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079834148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079834167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079834168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T05:16:14,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:14,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T05:16:14,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T05:16:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:14,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/c8030b45318245bdbeba1e87e0134d67 2024-11-20T05:16:14,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/fbac9d214c4d4a5f8f4a76216d466834 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fbac9d214c4d4a5f8f4a76216d466834 2024-11-20T05:16:14,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fbac9d214c4d4a5f8f4a76216d466834, entries=200, sequenceid=208, filesize=38.8 K 2024-11-20T05:16:14,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/712d58357e094b24bc1a77ce109a1c1e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/712d58357e094b24bc1a77ce109a1c1e 2024-11-20T05:16:14,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/712d58357e094b24bc1a77ce109a1c1e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-20T05:16:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/c8030b45318245bdbeba1e87e0134d67 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c8030b45318245bdbeba1e87e0134d67 2024-11-20T05:16:14,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c8030b45318245bdbeba1e87e0134d67, entries=150, sequenceid=208, filesize=11.9 K 2024-11-20T05:16:14,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e74402c4b2528f25097db0ff071d71d6 in 990ms, sequenceid=208, compaction requested=true 2024-11-20T05:16:14,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:14,454 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:14,454 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:14,455 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:14,455 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:14,455 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/A is initiating minor compaction (all files) 2024-11-20T05:16:14,455 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/B is initiating minor compaction (all files) 2024-11-20T05:16:14,456 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/A in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,456 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/B in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,456 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/79783fd16c0f448eaf28eee246f9d46e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/59616e60d52540eb99a640a0061c8787, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/712d58357e094b24bc1a77ce109a1c1e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=36.0 K 2024-11-20T05:16:14,456 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/a36abf1d36bd40dc9e0919af94bf8da9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5b8694ae0e034ebeb7d24100d17e0a53, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fbac9d214c4d4a5f8f4a76216d466834] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=100.0 K 2024-11-20T05:16:14,456 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,456 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/a36abf1d36bd40dc9e0919af94bf8da9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5b8694ae0e034ebeb7d24100d17e0a53, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fbac9d214c4d4a5f8f4a76216d466834] 2024-11-20T05:16:14,457 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 79783fd16c0f448eaf28eee246f9d46e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732079768528 2024-11-20T05:16:14,457 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36abf1d36bd40dc9e0919af94bf8da9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732079768528 2024-11-20T05:16:14,458 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 59616e60d52540eb99a640a0061c8787, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732079770084 2024-11-20T05:16:14,458 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b8694ae0e034ebeb7d24100d17e0a53, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732079770084 2024-11-20T05:16:14,459 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbac9d214c4d4a5f8f4a76216d466834, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732079772281 2024-11-20T05:16:14,459 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 712d58357e094b24bc1a77ce109a1c1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732079772281 2024-11-20T05:16:14,464 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:14,466 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:14,466 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/8c015dcd528e42439efc90fbaab0d0ae is 50, key is test_row_0/B:col10/1732079772298/Put/seqid=0 2024-11-20T05:16:14,468 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411205c6da78ccf7a4e8980734bf719f74e0c_e74402c4b2528f25097db0ff071d71d6 store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:14,470 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411205c6da78ccf7a4e8980734bf719f74e0c_e74402c4b2528f25097db0ff071d71d6, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:14,471 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205c6da78ccf7a4e8980734bf719f74e0c_e74402c4b2528f25097db0ff071d71d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742196_1372 (size=12663) 2024-11-20T05:16:14,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742197_1373 (size=4469) 2024-11-20T05:16:14,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T05:16:14,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,511 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T05:16:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:14,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204d9904d30f2a4d9a88e37bddd498aa14_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079773496/Put/seqid=0 2024-11-20T05:16:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T05:16:14,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742198_1374 (size=12304) 2024-11-20T05:16:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:14,538 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204d9904d30f2a4d9a88e37bddd498aa14_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204d9904d30f2a4d9a88e37bddd498aa14_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:14,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/4b5299d40e17473593296e5340a0c1c5, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:14,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/4b5299d40e17473593296e5340a0c1c5 is 175, key is test_row_0/A:col10/1732079773496/Put/seqid=0 2024-11-20T05:16:14,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742199_1375 (size=31105) 2024-11-20T05:16:14,559 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/4b5299d40e17473593296e5340a0c1c5 2024-11-20T05:16:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/45b72b40cf5a449398e0533b0179f073 is 50, key is test_row_0/B:col10/1732079773496/Put/seqid=0 2024-11-20T05:16:14,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742200_1376 (size=12151) 2024-11-20T05:16:14,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:14,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079834705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079834711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079834711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079834817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079834827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079834827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079834827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:14,883 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/8c015dcd528e42439efc90fbaab0d0ae as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/8c015dcd528e42439efc90fbaab0d0ae 2024-11-20T05:16:14,886 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/B of e74402c4b2528f25097db0ff071d71d6 into 8c015dcd528e42439efc90fbaab0d0ae(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:14,886 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:14,886 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/B, priority=13, startTime=1732079774454; duration=0sec 2024-11-20T05:16:14,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:14,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:B 2024-11-20T05:16:14,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:14,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:14,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/C is initiating minor compaction (all files) 2024-11-20T05:16:14,887 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/C in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:14,887 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0f618004dd574d0bba130fd72ebe7bf6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ce1eaedc610844428e83835930e5b1cd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c8030b45318245bdbeba1e87e0134d67] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=36.0 K 2024-11-20T05:16:14,888 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f618004dd574d0bba130fd72ebe7bf6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732079768528 2024-11-20T05:16:14,888 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1eaedc610844428e83835930e5b1cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732079770084 2024-11-20T05:16:14,888 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#A#compaction#311 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:14,888 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c8030b45318245bdbeba1e87e0134d67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732079772281 2024-11-20T05:16:14,889 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/048b5aa7d39b4b89a284a7e37887145c is 175, key is test_row_0/A:col10/1732079772298/Put/seqid=0 2024-11-20T05:16:14,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742201_1377 (size=31617) 2024-11-20T05:16:14,901 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:14,901 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/cce6315a54d145b1b7fb5cf56054d2f0 is 50, key is test_row_0/C:col10/1732079772298/Put/seqid=0 2024-11-20T05:16:14,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742202_1378 (size=12663) 2024-11-20T05:16:14,979 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/45b72b40cf5a449398e0533b0179f073 2024-11-20T05:16:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/04af702da2ab4fb7bdadb033e6b683b5 is 50, key is test_row_0/C:col10/1732079773496/Put/seqid=0 2024-11-20T05:16:14,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742203_1379 (size=12151) 2024-11-20T05:16:15,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079835027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T05:16:15,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079835038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079835038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079835038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,302 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/048b5aa7d39b4b89a284a7e37887145c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/048b5aa7d39b4b89a284a7e37887145c 2024-11-20T05:16:15,305 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/A of e74402c4b2528f25097db0ff071d71d6 into 048b5aa7d39b4b89a284a7e37887145c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:15,305 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:15,305 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/A, priority=13, startTime=1732079774454; duration=0sec 2024-11-20T05:16:15,306 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:15,306 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:A 2024-11-20T05:16:15,309 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/cce6315a54d145b1b7fb5cf56054d2f0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/cce6315a54d145b1b7fb5cf56054d2f0 2024-11-20T05:16:15,313 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/C of e74402c4b2528f25097db0ff071d71d6 into cce6315a54d145b1b7fb5cf56054d2f0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:15,313 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:15,313 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/C, priority=13, startTime=1732079774454; duration=0sec 2024-11-20T05:16:15,313 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:15,313 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:C 2024-11-20T05:16:15,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079835347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079835347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079835357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079835357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,398 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/04af702da2ab4fb7bdadb033e6b683b5 2024-11-20T05:16:15,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/4b5299d40e17473593296e5340a0c1c5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/4b5299d40e17473593296e5340a0c1c5 2024-11-20T05:16:15,404 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/4b5299d40e17473593296e5340a0c1c5, entries=150, sequenceid=223, filesize=30.4 K 2024-11-20T05:16:15,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/45b72b40cf5a449398e0533b0179f073 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/45b72b40cf5a449398e0533b0179f073 2024-11-20T05:16:15,407 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/45b72b40cf5a449398e0533b0179f073, entries=150, sequenceid=223, filesize=11.9 K 2024-11-20T05:16:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/04af702da2ab4fb7bdadb033e6b683b5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/04af702da2ab4fb7bdadb033e6b683b5 2024-11-20T05:16:15,411 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/04af702da2ab4fb7bdadb033e6b683b5, entries=150, sequenceid=223, filesize=11.9 K 2024-11-20T05:16:15,412 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e74402c4b2528f25097db0ff071d71d6 in 902ms, sequenceid=223, compaction requested=false 2024-11-20T05:16:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-20T05:16:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-20T05:16:15,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-20T05:16:15,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5120 sec 2024-11-20T05:16:15,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.5170 sec 2024-11-20T05:16:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:15,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T05:16:15,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:15,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:15,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:15,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:15,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:15,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:15,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112027fbb3efac3e471c84a48c58d7fc7ccf_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079774697/Put/seqid=0 2024-11-20T05:16:15,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742204_1380 (size=14794) 2024-11-20T05:16:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079835888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079835891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079835899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:15,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079835900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079836007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079836007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079836007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079836017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T05:16:16,047 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:16,047 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-20T05:16:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T05:16:16,049 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:16,049 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:16,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T05:16:16,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079836217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079836217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079836227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079836237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:16,295 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112027fbb3efac3e471c84a48c58d7fc7ccf_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027fbb3efac3e471c84a48c58d7fc7ccf_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:16,296 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/bcef585053e646c193cdc88a581574ba, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:16,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/bcef585053e646c193cdc88a581574ba is 175, key is test_row_0/A:col10/1732079774697/Put/seqid=0 2024-11-20T05:16:16,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742205_1381 (size=39749) 2024-11-20T05:16:16,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:16,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:16,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T05:16:16,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079836527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079836527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079836537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079836557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T05:16:16,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/bcef585053e646c193cdc88a581574ba 2024-11-20T05:16:16,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/a20f3557348f4d3da1050dc145cc06bc is 50, key is test_row_0/B:col10/1732079774697/Put/seqid=0 2024-11-20T05:16:16,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742206_1382 (size=12151) 2024-11-20T05:16:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37626 deadline: 1732079836787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:16,795 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in e74402c4b2528f25097db0ff071d71d6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:16,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:16,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079837037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:17,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079837047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079837057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:17,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079837077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:17,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/a20f3557348f4d3da1050dc145cc06bc 2024-11-20T05:16:17,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/0111fa388e624a238b07cd7d495665ac is 50, key is test_row_0/C:col10/1732079774697/Put/seqid=0 2024-11-20T05:16:17,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742207_1383 (size=12151) 2024-11-20T05:16:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T05:16:17,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:17,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/0111fa388e624a238b07cd7d495665ac 2024-11-20T05:16:17,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/bcef585053e646c193cdc88a581574ba as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/bcef585053e646c193cdc88a581574ba 2024-11-20T05:16:17,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/bcef585053e646c193cdc88a581574ba, entries=200, sequenceid=244, filesize=38.8 K 2024-11-20T05:16:17,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/a20f3557348f4d3da1050dc145cc06bc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/a20f3557348f4d3da1050dc145cc06bc 2024-11-20T05:16:17,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/a20f3557348f4d3da1050dc145cc06bc, entries=150, sequenceid=244, filesize=11.9 K 2024-11-20T05:16:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/0111fa388e624a238b07cd7d495665ac as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0111fa388e624a238b07cd7d495665ac 2024-11-20T05:16:17,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0111fa388e624a238b07cd7d495665ac, entries=150, sequenceid=244, filesize=11.9 K 2024-11-20T05:16:17,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e74402c4b2528f25097db0ff071d71d6 in 1671ms, sequenceid=244, compaction requested=true 2024-11-20T05:16:17,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:17,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:17,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:17,544 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:17,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:17,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:17,544 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:17,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:17,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:17,546 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:17,546 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:17,546 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/A is initiating minor compaction (all files) 2024-11-20T05:16:17,546 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/B is initiating minor compaction (all files) 2024-11-20T05:16:17,546 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/B in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,546 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/8c015dcd528e42439efc90fbaab0d0ae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/45b72b40cf5a449398e0533b0179f073, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/a20f3557348f4d3da1050dc145cc06bc] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=36.1 K 2024-11-20T05:16:17,546 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/A in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,546 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/048b5aa7d39b4b89a284a7e37887145c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/4b5299d40e17473593296e5340a0c1c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/bcef585053e646c193cdc88a581574ba] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=100.1 K 2024-11-20T05:16:17,546 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,546 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/048b5aa7d39b4b89a284a7e37887145c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/4b5299d40e17473593296e5340a0c1c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/bcef585053e646c193cdc88a581574ba] 2024-11-20T05:16:17,546 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c015dcd528e42439efc90fbaab0d0ae, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732079772281 2024-11-20T05:16:17,546 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048b5aa7d39b4b89a284a7e37887145c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732079772281 2024-11-20T05:16:17,547 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b72b40cf5a449398e0533b0179f073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732079773484 2024-11-20T05:16:17,547 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b5299d40e17473593296e5340a0c1c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732079773484 2024-11-20T05:16:17,547 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a20f3557348f4d3da1050dc145cc06bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732079774697 2024-11-20T05:16:17,547 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcef585053e646c193cdc88a581574ba, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732079774697 2024-11-20T05:16:17,553 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:17,558 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:17,559 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/7c993462e35d4fa7861209e77eac2609 is 50, key is test_row_0/B:col10/1732079774697/Put/seqid=0 2024-11-20T05:16:17,560 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112010d0259e97ab4f988ad4caa1ef544233_e74402c4b2528f25097db0ff071d71d6 store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:17,562 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112010d0259e97ab4f988ad4caa1ef544233_e74402c4b2528f25097db0ff071d71d6, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:17,562 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112010d0259e97ab4f988ad4caa1ef544233_e74402c4b2528f25097db0ff071d71d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:17,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T05:16:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,575 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T05:16:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:17,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742209_1385 (size=4469) 2024-11-20T05:16:17,584 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#A#compaction#320 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:17,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742208_1384 (size=12765) 2024-11-20T05:16:17,584 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/834b5a1e67704e7e82e3bfa3f90d1722 is 175, key is test_row_0/A:col10/1732079774697/Put/seqid=0 2024-11-20T05:16:17,592 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/7c993462e35d4fa7861209e77eac2609 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7c993462e35d4fa7861209e77eac2609 2024-11-20T05:16:17,597 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/B of e74402c4b2528f25097db0ff071d71d6 into 7c993462e35d4fa7861209e77eac2609(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:17,597 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:17,597 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/B, priority=13, startTime=1732079777544; duration=0sec 2024-11-20T05:16:17,598 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:17,598 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:B 2024-11-20T05:16:17,598 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:17,598 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:17,598 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/C is initiating minor compaction (all files) 2024-11-20T05:16:17,599 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/C in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:17,599 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/cce6315a54d145b1b7fb5cf56054d2f0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/04af702da2ab4fb7bdadb033e6b683b5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0111fa388e624a238b07cd7d495665ac] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=36.1 K 2024-11-20T05:16:17,599 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting cce6315a54d145b1b7fb5cf56054d2f0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732079772281 2024-11-20T05:16:17,599 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 04af702da2ab4fb7bdadb033e6b683b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732079773484 2024-11-20T05:16:17,599 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0111fa388e624a238b07cd7d495665ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732079774697 2024-11-20T05:16:17,607 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#C#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:17,608 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/520a65283c674984bffe44925b8a0a15 is 50, key is test_row_0/C:col10/1732079774697/Put/seqid=0 2024-11-20T05:16:17,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202e3be52baba24d73990e49fc1134840d_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079775899/Put/seqid=0 2024-11-20T05:16:17,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742210_1386 (size=31719) 2024-11-20T05:16:17,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/60446aa5adec4a5b9878d45ec4d7b273, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/6baa7a7086414c2bb2394b1fc1467b2c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d79a8cb4fc1e42708ec98bf53319a9fc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/32890ed9b31b4ed49e0502bd9fb73073, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3ed06974f2e844ea86038d53e1139c49, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/2ed482891c1d42dc9d229551d9b7478e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/80ee3a5ac807472c96354d7c9d2e78ec, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d2c1300e62114935a5549aee638cdb52, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/f39763ed369e4868b86e734de9a730a8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/1b0bd16f75794696a614151bd5ab5127, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fff1b375eaaf483ca97aed138c3e109a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5830240dd23f44c58bcb31540917a3b4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/a36abf1d36bd40dc9e0919af94bf8da9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5b8694ae0e034ebeb7d24100d17e0a53, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fbac9d214c4d4a5f8f4a76216d466834] to archive 2024-11-20T05:16:17,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:17,637 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/60446aa5adec4a5b9878d45ec4d7b273 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/60446aa5adec4a5b9878d45ec4d7b273 2024-11-20T05:16:17,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/6baa7a7086414c2bb2394b1fc1467b2c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/6baa7a7086414c2bb2394b1fc1467b2c 2024-11-20T05:16:17,646 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d79a8cb4fc1e42708ec98bf53319a9fc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d79a8cb4fc1e42708ec98bf53319a9fc 2024-11-20T05:16:17,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/32890ed9b31b4ed49e0502bd9fb73073 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/32890ed9b31b4ed49e0502bd9fb73073 2024-11-20T05:16:17,654 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3ed06974f2e844ea86038d53e1139c49 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3ed06974f2e844ea86038d53e1139c49 2024-11-20T05:16:17,655 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/2ed482891c1d42dc9d229551d9b7478e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/2ed482891c1d42dc9d229551d9b7478e 2024-11-20T05:16:17,656 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/80ee3a5ac807472c96354d7c9d2e78ec to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/80ee3a5ac807472c96354d7c9d2e78ec 2024-11-20T05:16:17,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d2c1300e62114935a5549aee638cdb52 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/d2c1300e62114935a5549aee638cdb52 2024-11-20T05:16:17,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/f39763ed369e4868b86e734de9a730a8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/f39763ed369e4868b86e734de9a730a8 2024-11-20T05:16:17,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/1b0bd16f75794696a614151bd5ab5127 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/1b0bd16f75794696a614151bd5ab5127 2024-11-20T05:16:17,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fff1b375eaaf483ca97aed138c3e109a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fff1b375eaaf483ca97aed138c3e109a 2024-11-20T05:16:17,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5830240dd23f44c58bcb31540917a3b4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5830240dd23f44c58bcb31540917a3b4 2024-11-20T05:16:17,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/a36abf1d36bd40dc9e0919af94bf8da9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/a36abf1d36bd40dc9e0919af94bf8da9 2024-11-20T05:16:17,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5b8694ae0e034ebeb7d24100d17e0a53 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/5b8694ae0e034ebeb7d24100d17e0a53 2024-11-20T05:16:17,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742212_1388 (size=12765) 2024-11-20T05:16:17,676 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/520a65283c674984bffe44925b8a0a15 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/520a65283c674984bffe44925b8a0a15 2024-11-20T05:16:17,677 WARN [regionserver/51d5d2011a1c:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-20T05:16:17,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742211_1387 (size=12354) 2024-11-20T05:16:17,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:17,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fbac9d214c4d4a5f8f4a76216d466834 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/fbac9d214c4d4a5f8f4a76216d466834 2024-11-20T05:16:17,683 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/C of e74402c4b2528f25097db0ff071d71d6 into 520a65283c674984bffe44925b8a0a15(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:17,683 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:17,683 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/C, priority=13, startTime=1732079777544; duration=0sec 2024-11-20T05:16:17,684 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:17,684 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:C 2024-11-20T05:16:17,684 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202e3be52baba24d73990e49fc1134840d_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202e3be52baba24d73990e49fc1134840d_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/eafff05246c9414181d5a39f24bff556, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/eafff05246c9414181d5a39f24bff556 is 175, key is test_row_0/A:col10/1732079775899/Put/seqid=0 2024-11-20T05:16:17,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/fe757a0ed3d94907ba7047832a5645cc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/1eeacb0f3d3645fab662f03f276d245c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/e18963ed21f74d6896880974a5dc079f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/471b349bc5cc4f5dbcac4e97923ce47b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/b2ad5260ef5046e689398a3eb3475e67, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d3c2aad1defa4fbbbbb068dd77e42267, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/9b51e461928647e9acd23e9a5972c6a6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d8390c42311845d0bc189a9576029b3b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/4969496eb9ed457f94f074220b5d9166, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7e62ecf3df1e4c608b65c6d9ca94a90e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d2c5bd62eaef4059a1c4ead3b7e79ac5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/79783fd16c0f448eaf28eee246f9d46e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c3c9dc8131f54e1780ed3a71d6b219c0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/59616e60d52540eb99a640a0061c8787, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/8c015dcd528e42439efc90fbaab0d0ae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/712d58357e094b24bc1a77ce109a1c1e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/45b72b40cf5a449398e0533b0179f073, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/a20f3557348f4d3da1050dc145cc06bc] to archive 2024-11-20T05:16:17,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742213_1389 (size=31155) 2024-11-20T05:16:17,696 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/eafff05246c9414181d5a39f24bff556 2024-11-20T05:16:17,697 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:17,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/fe757a0ed3d94907ba7047832a5645cc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/fe757a0ed3d94907ba7047832a5645cc 2024-11-20T05:16:17,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/1eeacb0f3d3645fab662f03f276d245c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/1eeacb0f3d3645fab662f03f276d245c 2024-11-20T05:16:17,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/e18963ed21f74d6896880974a5dc079f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/e18963ed21f74d6896880974a5dc079f 2024-11-20T05:16:17,705 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/471b349bc5cc4f5dbcac4e97923ce47b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/471b349bc5cc4f5dbcac4e97923ce47b 2024-11-20T05:16:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/43cc4d6ac9e64b69bbb9780bc343abc9 is 50, key is test_row_0/B:col10/1732079775899/Put/seqid=0 2024-11-20T05:16:17,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/b2ad5260ef5046e689398a3eb3475e67 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/b2ad5260ef5046e689398a3eb3475e67 2024-11-20T05:16:17,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d3c2aad1defa4fbbbbb068dd77e42267 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d3c2aad1defa4fbbbbb068dd77e42267 2024-11-20T05:16:17,711 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/9b51e461928647e9acd23e9a5972c6a6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/9b51e461928647e9acd23e9a5972c6a6 2024-11-20T05:16:17,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742214_1390 (size=12201) 2024-11-20T05:16:17,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d8390c42311845d0bc189a9576029b3b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d8390c42311845d0bc189a9576029b3b 2024-11-20T05:16:17,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/4969496eb9ed457f94f074220b5d9166 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/4969496eb9ed457f94f074220b5d9166 2024-11-20T05:16:17,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7e62ecf3df1e4c608b65c6d9ca94a90e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7e62ecf3df1e4c608b65c6d9ca94a90e 2024-11-20T05:16:17,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d2c5bd62eaef4059a1c4ead3b7e79ac5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/d2c5bd62eaef4059a1c4ead3b7e79ac5 2024-11-20T05:16:17,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/79783fd16c0f448eaf28eee246f9d46e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/79783fd16c0f448eaf28eee246f9d46e 2024-11-20T05:16:17,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c3c9dc8131f54e1780ed3a71d6b219c0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c3c9dc8131f54e1780ed3a71d6b219c0 2024-11-20T05:16:17,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/59616e60d52540eb99a640a0061c8787 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/59616e60d52540eb99a640a0061c8787 2024-11-20T05:16:17,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/8c015dcd528e42439efc90fbaab0d0ae to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/8c015dcd528e42439efc90fbaab0d0ae 2024-11-20T05:16:17,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/712d58357e094b24bc1a77ce109a1c1e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/712d58357e094b24bc1a77ce109a1c1e 2024-11-20T05:16:17,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/45b72b40cf5a449398e0533b0179f073 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/45b72b40cf5a449398e0533b0179f073 2024-11-20T05:16:17,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/a20f3557348f4d3da1050dc145cc06bc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/a20f3557348f4d3da1050dc145cc06bc 2024-11-20T05:16:17,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5e597fafcc8c40e3b49460675e840102, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/af5df4a979dc4685b35eaf80992b1fe8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/3a56126202554ad2ae84b2b906dbd191, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ea600d02d5964189aa12d1c1ecf33952, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/38c789d3dcdc43a1826115baee1fed04, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/111ccb13c6dc404588c33df1d2f9831a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/7f2e7839aac4402c8c1d28cfe2ee40c6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/be4a680d9ae54c9b9e02894ade8413fe, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/96c7591bb5224c71ad197aa2c5076d4b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c03665d8bc184087b24b779a2678e79c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/50cf9032e48c46e585c0da0a92e3fe8b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0f618004dd574d0bba130fd72ebe7bf6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5aed998b0a2448a3b23f448d43e7425b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ce1eaedc610844428e83835930e5b1cd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/cce6315a54d145b1b7fb5cf56054d2f0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c8030b45318245bdbeba1e87e0134d67, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/04af702da2ab4fb7bdadb033e6b683b5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0111fa388e624a238b07cd7d495665ac] to archive 2024-11-20T05:16:17,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:17,736 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5e597fafcc8c40e3b49460675e840102 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5e597fafcc8c40e3b49460675e840102 2024-11-20T05:16:17,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/af5df4a979dc4685b35eaf80992b1fe8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/af5df4a979dc4685b35eaf80992b1fe8 2024-11-20T05:16:17,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/3a56126202554ad2ae84b2b906dbd191 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/3a56126202554ad2ae84b2b906dbd191 2024-11-20T05:16:17,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ea600d02d5964189aa12d1c1ecf33952 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ea600d02d5964189aa12d1c1ecf33952 2024-11-20T05:16:17,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/38c789d3dcdc43a1826115baee1fed04 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/38c789d3dcdc43a1826115baee1fed04 2024-11-20T05:16:17,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/111ccb13c6dc404588c33df1d2f9831a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/111ccb13c6dc404588c33df1d2f9831a 2024-11-20T05:16:17,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/7f2e7839aac4402c8c1d28cfe2ee40c6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/7f2e7839aac4402c8c1d28cfe2ee40c6 2024-11-20T05:16:17,747 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/be4a680d9ae54c9b9e02894ade8413fe to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/be4a680d9ae54c9b9e02894ade8413fe 2024-11-20T05:16:17,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/96c7591bb5224c71ad197aa2c5076d4b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/96c7591bb5224c71ad197aa2c5076d4b 2024-11-20T05:16:17,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c03665d8bc184087b24b779a2678e79c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c03665d8bc184087b24b779a2678e79c 2024-11-20T05:16:17,753 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/50cf9032e48c46e585c0da0a92e3fe8b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/50cf9032e48c46e585c0da0a92e3fe8b 2024-11-20T05:16:17,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0f618004dd574d0bba130fd72ebe7bf6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0f618004dd574d0bba130fd72ebe7bf6 2024-11-20T05:16:17,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5aed998b0a2448a3b23f448d43e7425b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5aed998b0a2448a3b23f448d43e7425b 2024-11-20T05:16:17,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ce1eaedc610844428e83835930e5b1cd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/ce1eaedc610844428e83835930e5b1cd 2024-11-20T05:16:17,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/cce6315a54d145b1b7fb5cf56054d2f0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/cce6315a54d145b1b7fb5cf56054d2f0 2024-11-20T05:16:17,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c8030b45318245bdbeba1e87e0134d67 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/c8030b45318245bdbeba1e87e0134d67 2024-11-20T05:16:17,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/04af702da2ab4fb7bdadb033e6b683b5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/04af702da2ab4fb7bdadb033e6b683b5 2024-11-20T05:16:17,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51d5d2011a1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0111fa388e624a238b07cd7d495665ac to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0111fa388e624a238b07cd7d495665ac 2024-11-20T05:16:18,034 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/834b5a1e67704e7e82e3bfa3f90d1722 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/834b5a1e67704e7e82e3bfa3f90d1722 2024-11-20T05:16:18,040 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/A of e74402c4b2528f25097db0ff071d71d6 into 834b5a1e67704e7e82e3bfa3f90d1722(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:18,040 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:18,040 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/A, priority=13, startTime=1732079777544; duration=0sec 2024-11-20T05:16:18,040 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:18,040 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:A 2024-11-20T05:16:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:18,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:18,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079838099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,115 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/43cc4d6ac9e64b69bbb9780bc343abc9 2024-11-20T05:16:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079838103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079838105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079838103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5658bacd204640dfab8d1bc7e967340b is 50, key is test_row_0/C:col10/1732079775899/Put/seqid=0 2024-11-20T05:16:18,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742215_1391 (size=12201) 2024-11-20T05:16:18,142 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5658bacd204640dfab8d1bc7e967340b 2024-11-20T05:16:18,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/eafff05246c9414181d5a39f24bff556 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/eafff05246c9414181d5a39f24bff556 2024-11-20T05:16:18,162 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/eafff05246c9414181d5a39f24bff556, entries=150, sequenceid=258, filesize=30.4 K 2024-11-20T05:16:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/43cc4d6ac9e64b69bbb9780bc343abc9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/43cc4d6ac9e64b69bbb9780bc343abc9 2024-11-20T05:16:18,169 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/43cc4d6ac9e64b69bbb9780bc343abc9, entries=150, sequenceid=258, filesize=11.9 K 2024-11-20T05:16:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/5658bacd204640dfab8d1bc7e967340b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5658bacd204640dfab8d1bc7e967340b 2024-11-20T05:16:18,174 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5658bacd204640dfab8d1bc7e967340b, entries=150, sequenceid=258, filesize=11.9 K 2024-11-20T05:16:18,175 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e74402c4b2528f25097db0ff071d71d6 in 600ms, sequenceid=258, compaction requested=false 2024-11-20T05:16:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-20T05:16:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-20T05:16:18,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-20T05:16:18,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1280 sec 2024-11-20T05:16:18,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1310 sec 2024-11-20T05:16:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T05:16:18,208 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:18,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-20T05:16:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T05:16:18,209 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:18,210 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:18,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:18,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T05:16:18,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:18,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203f203787bd1b46d793dfc36a7b1a9bdd_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079778103/Put/seqid=0 2024-11-20T05:16:18,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742216_1392 (size=14994) 2024-11-20T05:16:18,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079838250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079838251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079838251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079838255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T05:16:18,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079838367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079838367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079838367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079838368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:18,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T05:16:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079838577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079838577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079838578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079838578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:18,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203f203787bd1b46d793dfc36a7b1a9bdd_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f203787bd1b46d793dfc36a7b1a9bdd_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:18,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/230efe2e40064843aaaafef3c9f8819f, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:18,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/230efe2e40064843aaaafef3c9f8819f is 175, key is test_row_0/A:col10/1732079778103/Put/seqid=0 2024-11-20T05:16:18,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742217_1393 (size=39949) 2024-11-20T05:16:18,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:18,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:18,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T05:16:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079838897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079838897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079838897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079838898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:18,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:18,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:18,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:18,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/230efe2e40064843aaaafef3c9f8819f 2024-11-20T05:16:19,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/641d6d68959645e68762bbd2e6c5657d is 50, key is test_row_0/B:col10/1732079778103/Put/seqid=0 2024-11-20T05:16:19,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742218_1394 (size=12301) 2024-11-20T05:16:19,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:19,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:19,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,257 INFO [Thread-1455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1455. 2024-11-20T05:16:19,257 DEBUG [Thread-1455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:19,257 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,257 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:19,258 INFO [Thread-1453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1453. 2024-11-20T05:16:19,258 DEBUG [Thread-1453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:19,258 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,258 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:19,262 INFO [Thread-1451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1451. 2024-11-20T05:16:19,262 DEBUG [Thread-1451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:19,262 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,262 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:19,271 INFO [Thread-1447 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1447. 2024-11-20T05:16:19,271 DEBUG [Thread-1447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:19,271 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,271 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:19,272 INFO [Thread-1449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1449. 2024-11-20T05:16:19,272 DEBUG [Thread-1449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:19,272 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,272 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:19,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:19,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T05:16:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37616 deadline: 1732079839408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37642 deadline: 1732079839408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37656 deadline: 1732079839408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1732079839417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:19,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/641d6d68959645e68762bbd2e6c5657d 2024-11-20T05:16:19,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/dc1e7821ce54408192c1fe55fae9b395 is 50, key is test_row_0/C:col10/1732079778103/Put/seqid=0 2024-11-20T05:16:19,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742219_1395 (size=12301) 2024-11-20T05:16:19,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:19,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:19,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:19,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/dc1e7821ce54408192c1fe55fae9b395 2024-11-20T05:16:19,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/230efe2e40064843aaaafef3c9f8819f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/230efe2e40064843aaaafef3c9f8819f 2024-11-20T05:16:19,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/230efe2e40064843aaaafef3c9f8819f, entries=200, sequenceid=280, filesize=39.0 K 2024-11-20T05:16:19,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/641d6d68959645e68762bbd2e6c5657d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/641d6d68959645e68762bbd2e6c5657d 2024-11-20T05:16:19,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/641d6d68959645e68762bbd2e6c5657d, entries=150, sequenceid=280, filesize=12.0 K 2024-11-20T05:16:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/dc1e7821ce54408192c1fe55fae9b395 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/dc1e7821ce54408192c1fe55fae9b395 2024-11-20T05:16:19,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/dc1e7821ce54408192c1fe55fae9b395, entries=150, sequenceid=280, filesize=12.0 K 2024-11-20T05:16:19,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e74402c4b2528f25097db0ff071d71d6 in 1660ms, sequenceid=280, compaction requested=true 2024-11-20T05:16:19,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:19,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:19,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:19,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:19,886 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:19,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:19,886 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:19,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e74402c4b2528f25097db0ff071d71d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:19,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:19,887 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:19,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:19,887 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/A is initiating minor compaction (all files) 2024-11-20T05:16:19,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/B is initiating minor compaction (all files) 2024-11-20T05:16:19,887 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/A in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,887 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/B in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,887 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/834b5a1e67704e7e82e3bfa3f90d1722, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/eafff05246c9414181d5a39f24bff556, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/230efe2e40064843aaaafef3c9f8819f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=100.4 K 2024-11-20T05:16:19,887 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7c993462e35d4fa7861209e77eac2609, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/43cc4d6ac9e64b69bbb9780bc343abc9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/641d6d68959645e68762bbd2e6c5657d] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=36.4 K 2024-11-20T05:16:19,887 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,887 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/834b5a1e67704e7e82e3bfa3f90d1722, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/eafff05246c9414181d5a39f24bff556, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/230efe2e40064843aaaafef3c9f8819f] 2024-11-20T05:16:19,888 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c993462e35d4fa7861209e77eac2609, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732079774697 2024-11-20T05:16:19,888 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834b5a1e67704e7e82e3bfa3f90d1722, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732079774697 2024-11-20T05:16:19,888 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 43cc4d6ac9e64b69bbb9780bc343abc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732079775874 2024-11-20T05:16:19,888 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting eafff05246c9414181d5a39f24bff556, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732079775874 2024-11-20T05:16:19,888 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 641d6d68959645e68762bbd2e6c5657d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732079778095 2024-11-20T05:16:19,888 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 230efe2e40064843aaaafef3c9f8819f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732079778095 2024-11-20T05:16:19,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T05:16:19,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:19,891 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T05:16:19,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:19,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:19,892 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:19,894 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:19,894 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411201556cfedceec43c9be69ad3975dfc85a_e74402c4b2528f25097db0ff071d71d6 store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:19,894 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/c842b073bcb443e6ad202b535e1beee9 is 50, key is test_row_0/B:col10/1732079778103/Put/seqid=0 2024-11-20T05:16:19,898 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411201556cfedceec43c9be69ad3975dfc85a_e74402c4b2528f25097db0ff071d71d6, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b76e141b5bd846bc887bd675dd7be8e6_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_0/A:col10/1732079778234/Put/seqid=0 2024-11-20T05:16:19,898 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201556cfedceec43c9be69ad3975dfc85a_e74402c4b2528f25097db0ff071d71d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:19,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742220_1396 (size=12404) 2024-11-20T05:16:19,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742221_1397 (size=4469) 2024-11-20T05:16:19,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742222_1398 (size=12454) 2024-11-20T05:16:20,306 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:20,307 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/3c1d06bcf78b44ceb2892f41385ff616 is 175, key is test_row_0/A:col10/1732079778103/Put/seqid=0 2024-11-20T05:16:20,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T05:16:20,360 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b76e141b5bd846bc887bd675dd7be8e6_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b76e141b5bd846bc887bd675dd7be8e6_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:20,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742223_1399 (size=31461) 2024-11-20T05:16:20,361 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/c842b073bcb443e6ad202b535e1beee9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c842b073bcb443e6ad202b535e1beee9 2024-11-20T05:16:20,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/e9f2a2edb69e429abbf2efbd646eab88, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:20,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/e9f2a2edb69e429abbf2efbd646eab88 is 175, key is test_row_0/A:col10/1732079778234/Put/seqid=0 2024-11-20T05:16:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742224_1400 (size=31255) 2024-11-20T05:16:20,367 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/3c1d06bcf78b44ceb2892f41385ff616 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3c1d06bcf78b44ceb2892f41385ff616 2024-11-20T05:16:20,367 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/B of e74402c4b2528f25097db0ff071d71d6 into c842b073bcb443e6ad202b535e1beee9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:20,367 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:20,367 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/B, priority=13, startTime=1732079779886; duration=0sec 2024-11-20T05:16:20,367 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:20,367 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:B 2024-11-20T05:16:20,367 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:20,368 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:20,368 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): e74402c4b2528f25097db0ff071d71d6/C is initiating minor compaction (all files) 2024-11-20T05:16:20,368 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e74402c4b2528f25097db0ff071d71d6/C in TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:20,368 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/520a65283c674984bffe44925b8a0a15, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5658bacd204640dfab8d1bc7e967340b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/dc1e7821ce54408192c1fe55fae9b395] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp, totalSize=36.4 K 2024-11-20T05:16:20,368 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 520a65283c674984bffe44925b8a0a15, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732079774697 2024-11-20T05:16:20,369 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5658bacd204640dfab8d1bc7e967340b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732079775874 2024-11-20T05:16:20,369 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting dc1e7821ce54408192c1fe55fae9b395, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732079778095 2024-11-20T05:16:20,370 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/A of e74402c4b2528f25097db0ff071d71d6 into 3c1d06bcf78b44ceb2892f41385ff616(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:20,370 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:20,370 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/A, priority=13, startTime=1732079779886; duration=0sec 2024-11-20T05:16:20,371 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:20,371 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:A 2024-11-20T05:16:20,374 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e74402c4b2528f25097db0ff071d71d6#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:20,375 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/0a3eae17aa6f47c0abdefdf7a2ccfc16 is 50, key is test_row_0/C:col10/1732079778103/Put/seqid=0 2024-11-20T05:16:20,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742225_1401 (size=12404) 2024-11-20T05:16:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:20,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. as already flushing 2024-11-20T05:16:20,419 INFO [Thread-1440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1440. 2024-11-20T05:16:20,419 DEBUG [Thread-1440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:20,419 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,419 INFO [Thread-1442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1442. 2024-11-20T05:16:20,419 INFO [Thread-1436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1436. 2024-11-20T05:16:20,419 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,419 DEBUG [Thread-1442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:20,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:20,420 DEBUG [Thread-1436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:20,420 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,420 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,420 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:20,420 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:20,429 INFO [Thread-1444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1444. 2024-11-20T05:16:20,429 DEBUG [Thread-1444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:20,429 DEBUG [Thread-1444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,429 DEBUG [Thread-1444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:20,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:20,767 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/e9f2a2edb69e429abbf2efbd646eab88 2024-11-20T05:16:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/acee8cc7231d4e1dadf89d4761fe6c80 is 50, key is test_row_0/B:col10/1732079778234/Put/seqid=0 2024-11-20T05:16:20,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742226_1402 (size=12301) 2024-11-20T05:16:20,783 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/0a3eae17aa6f47c0abdefdf7a2ccfc16 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0a3eae17aa6f47c0abdefdf7a2ccfc16 2024-11-20T05:16:20,787 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e74402c4b2528f25097db0ff071d71d6/C of e74402c4b2528f25097db0ff071d71d6 into 0a3eae17aa6f47c0abdefdf7a2ccfc16(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:20,787 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:20,788 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., storeName=e74402c4b2528f25097db0ff071d71d6/C, priority=13, startTime=1732079779886; duration=0sec 2024-11-20T05:16:20,788 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:20,788 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e74402c4b2528f25097db0ff071d71d6:C 2024-11-20T05:16:21,182 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/acee8cc7231d4e1dadf89d4761fe6c80 2024-11-20T05:16:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/72f09ef8878147bc984a69baadcd99bd is 50, key is test_row_0/C:col10/1732079778234/Put/seqid=0 2024-11-20T05:16:21,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742227_1403 (size=12301) 2024-11-20T05:16:21,601 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/72f09ef8878147bc984a69baadcd99bd 2024-11-20T05:16:21,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/e9f2a2edb69e429abbf2efbd646eab88 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/e9f2a2edb69e429abbf2efbd646eab88 2024-11-20T05:16:21,613 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/e9f2a2edb69e429abbf2efbd646eab88, entries=150, sequenceid=293, filesize=30.5 K 2024-11-20T05:16:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/acee8cc7231d4e1dadf89d4761fe6c80 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/acee8cc7231d4e1dadf89d4761fe6c80 2024-11-20T05:16:21,617 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/acee8cc7231d4e1dadf89d4761fe6c80, entries=150, sequenceid=293, filesize=12.0 K 2024-11-20T05:16:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/72f09ef8878147bc984a69baadcd99bd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/72f09ef8878147bc984a69baadcd99bd 2024-11-20T05:16:21,621 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/72f09ef8878147bc984a69baadcd99bd, entries=150, sequenceid=293, filesize=12.0 K 2024-11-20T05:16:21,621 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for e74402c4b2528f25097db0ff071d71d6 in 1730ms, sequenceid=293, compaction requested=false 2024-11-20T05:16:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for e74402c4b2528f25097db0ff071d71d6: 2024-11-20T05:16:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-20T05:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-20T05:16:21,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-20T05:16:21,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4120 sec 2024-11-20T05:16:21,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.4160 sec 2024-11-20T05:16:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T05:16:22,368 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:26,850 INFO [Thread-1438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1438. 2024-11-20T05:16:26,850 DEBUG [Thread-1438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:26,850 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:26,850 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:26,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T05:16:26,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1066 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3185 rows 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1065 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3181 rows 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1056 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3168 rows 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1056 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3168 rows 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-20T05:16:26,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3174 rows 2024-11-20T05:16:26,851 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:16:26,851 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:16:26,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:26,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:26,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:26,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T05:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T05:16:26,859 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079786859"}]},"ts":"1732079786859"} 2024-11-20T05:16:26,860 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T05:16:26,860 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T05:16:26,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:16:26,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, UNASSIGN}] 2024-11-20T05:16:26,863 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, UNASSIGN 2024-11-20T05:16:26,864 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:26,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, UNASSIGN because future has completed 2024-11-20T05:16:26,866 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:16:26,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T05:16:27,018 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:27,018 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:16:27,018 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing e74402c4b2528f25097db0ff071d71d6, disabling compactions & flushes 2024-11-20T05:16:27,018 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. after waiting 0 ms 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:27,019 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing e74402c4b2528f25097db0ff071d71d6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=A 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=B 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e74402c4b2528f25097db0ff071d71d6, store=C 2024-11-20T05:16:27,019 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:27,024 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112067f3219f3f4e408dad42ef0c6db047a9_e74402c4b2528f25097db0ff071d71d6 is 50, key is test_row_1/A:col10/1732079786848/Put/seqid=0 2024-11-20T05:16:27,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742228_1404 (size=9914) 2024-11-20T05:16:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T05:16:27,427 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:27,432 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112067f3219f3f4e408dad42ef0c6db047a9_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112067f3219f3f4e408dad42ef0c6db047a9_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:27,432 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/0c30474a37b24da7a1a9815af80c06b3, store: [table=TestAcidGuarantees family=A region=e74402c4b2528f25097db0ff071d71d6] 2024-11-20T05:16:27,433 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/0c30474a37b24da7a1a9815af80c06b3 is 175, key is test_row_1/A:col10/1732079786848/Put/seqid=0 2024-11-20T05:16:27,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742229_1405 (size=22561) 2024-11-20T05:16:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T05:16:27,838 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/0c30474a37b24da7a1a9815af80c06b3 2024-11-20T05:16:27,852 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/89fab725657b4008bc1d33b4d2dbdee9 is 50, key is test_row_1/B:col10/1732079786848/Put/seqid=0 2024-11-20T05:16:27,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742230_1406 (size=9857) 2024-11-20T05:16:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T05:16:28,277 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/89fab725657b4008bc1d33b4d2dbdee9 2024-11-20T05:16:28,283 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/f89b87c7d958405dab26393212394f8c is 50, key is test_row_1/C:col10/1732079786848/Put/seqid=0 2024-11-20T05:16:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742231_1407 (size=9857) 2024-11-20T05:16:28,688 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/f89b87c7d958405dab26393212394f8c 2024-11-20T05:16:28,697 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/A/0c30474a37b24da7a1a9815af80c06b3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/0c30474a37b24da7a1a9815af80c06b3 2024-11-20T05:16:28,703 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/0c30474a37b24da7a1a9815af80c06b3, entries=100, sequenceid=304, filesize=22.0 K 2024-11-20T05:16:28,704 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/B/89fab725657b4008bc1d33b4d2dbdee9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/89fab725657b4008bc1d33b4d2dbdee9 2024-11-20T05:16:28,708 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/89fab725657b4008bc1d33b4d2dbdee9, entries=100, sequenceid=304, filesize=9.6 K 2024-11-20T05:16:28,709 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/.tmp/C/f89b87c7d958405dab26393212394f8c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/f89b87c7d958405dab26393212394f8c 2024-11-20T05:16:28,712 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/f89b87c7d958405dab26393212394f8c, entries=100, sequenceid=304, filesize=9.6 K 2024-11-20T05:16:28,712 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e74402c4b2528f25097db0ff071d71d6 in 1693ms, sequenceid=304, compaction requested=true 2024-11-20T05:16:28,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/048b5aa7d39b4b89a284a7e37887145c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/4b5299d40e17473593296e5340a0c1c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/bcef585053e646c193cdc88a581574ba, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/834b5a1e67704e7e82e3bfa3f90d1722, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/eafff05246c9414181d5a39f24bff556, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/230efe2e40064843aaaafef3c9f8819f] to archive 2024-11-20T05:16:28,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:28,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/048b5aa7d39b4b89a284a7e37887145c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/048b5aa7d39b4b89a284a7e37887145c 2024-11-20T05:16:28,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/4b5299d40e17473593296e5340a0c1c5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/4b5299d40e17473593296e5340a0c1c5 2024-11-20T05:16:28,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/bcef585053e646c193cdc88a581574ba to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/bcef585053e646c193cdc88a581574ba 2024-11-20T05:16:28,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/834b5a1e67704e7e82e3bfa3f90d1722 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/834b5a1e67704e7e82e3bfa3f90d1722 2024-11-20T05:16:28,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/eafff05246c9414181d5a39f24bff556 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/eafff05246c9414181d5a39f24bff556 2024-11-20T05:16:28,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/230efe2e40064843aaaafef3c9f8819f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/230efe2e40064843aaaafef3c9f8819f 2024-11-20T05:16:28,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7c993462e35d4fa7861209e77eac2609, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/43cc4d6ac9e64b69bbb9780bc343abc9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/641d6d68959645e68762bbd2e6c5657d] to archive 2024-11-20T05:16:28,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:28,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7c993462e35d4fa7861209e77eac2609 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/7c993462e35d4fa7861209e77eac2609 2024-11-20T05:16:28,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/43cc4d6ac9e64b69bbb9780bc343abc9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/43cc4d6ac9e64b69bbb9780bc343abc9 2024-11-20T05:16:28,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/641d6d68959645e68762bbd2e6c5657d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/641d6d68959645e68762bbd2e6c5657d 2024-11-20T05:16:28,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/520a65283c674984bffe44925b8a0a15, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5658bacd204640dfab8d1bc7e967340b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/dc1e7821ce54408192c1fe55fae9b395] to archive 2024-11-20T05:16:28,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:28,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/520a65283c674984bffe44925b8a0a15 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/520a65283c674984bffe44925b8a0a15 2024-11-20T05:16:28,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5658bacd204640dfab8d1bc7e967340b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/5658bacd204640dfab8d1bc7e967340b 2024-11-20T05:16:28,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/dc1e7821ce54408192c1fe55fae9b395 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/dc1e7821ce54408192c1fe55fae9b395 2024-11-20T05:16:28,733 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=4 2024-11-20T05:16:28,734 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. 2024-11-20T05:16:28,734 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for e74402c4b2528f25097db0ff071d71d6: Waiting for close lock at 1732079787018Running coprocessor pre-close hooks at 1732079787018Disabling compacts and flushes for region at 1732079787018Disabling writes for close at 1732079787019 (+1 ms)Obtaining lock to block concurrent updates at 1732079787019Preparing flush snapshotting stores in e74402c4b2528f25097db0ff071d71d6 at 1732079787019Finished memstore snapshotting TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732079787019Flushing stores of TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6. at 1732079787019Flushing e74402c4b2528f25097db0ff071d71d6/A: creating writer at 1732079787019Flushing mob file e74402c4b2528f25097db0ff071d71d6/A: appending metadata at 1732079787023 (+4 ms)Flushing mob file e74402c4b2528f25097db0ff071d71d6/A: closing flushed file at 1732079787023Flushing e74402c4b2528f25097db0ff071d71d6/A: appending metadata at 1732079787433 (+410 ms)Flushing e74402c4b2528f25097db0ff071d71d6/A: closing flushed file at 1732079787433Flushing e74402c4b2528f25097db0ff071d71d6/B: creating writer at 1732079787847 (+414 ms)Flushing e74402c4b2528f25097db0ff071d71d6/B: appending metadata at 1732079787851 (+4 ms)Flushing e74402c4b2528f25097db0ff071d71d6/B: closing flushed file at 1732079787851Flushing e74402c4b2528f25097db0ff071d71d6/C: creating writer at 1732079788280 (+429 ms)Flushing e74402c4b2528f25097db0ff071d71d6/C: appending metadata at 1732079788282 (+2 ms)Flushing e74402c4b2528f25097db0ff071d71d6/C: closing flushed file at 1732079788282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f653bb: reopening flushed file at 1732079788696 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e9d466e: reopening flushed file at 1732079788703 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59a4d213: reopening flushed file at 1732079788708 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e74402c4b2528f25097db0ff071d71d6 in 1693ms, sequenceid=304, compaction requested=true at 1732079788712 (+4 ms)Writing region close event to WAL at 1732079788730 (+18 ms)Running coprocessor post-close hooks at 1732079788733 (+3 ms)Closed at 1732079788734 (+1 ms) 2024-11-20T05:16:28,736 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:28,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e74402c4b2528f25097db0ff071d71d6, regionState=CLOSED 2024-11-20T05:16:28,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:16:28,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T05:16:28,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure e74402c4b2528f25097db0ff071d71d6, server=51d5d2011a1c,44753,1732079655401 in 1.8720 sec 2024-11-20T05:16:28,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-20T05:16:28,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e74402c4b2528f25097db0ff071d71d6, UNASSIGN in 1.8770 sec 2024-11-20T05:16:28,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-20T05:16:28,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8800 sec 2024-11-20T05:16:28,743 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079788743"}]},"ts":"1732079788743"} 2024-11-20T05:16:28,744 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T05:16:28,744 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T05:16:28,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8920 sec 2024-11-20T05:16:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T05:16:29,008 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:29,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T05:16:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:29,010 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T05:16:29,011 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:29,014 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/recovered.edits] 2024-11-20T05:16:29,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/0c30474a37b24da7a1a9815af80c06b3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/0c30474a37b24da7a1a9815af80c06b3 2024-11-20T05:16:29,020 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3c1d06bcf78b44ceb2892f41385ff616 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/3c1d06bcf78b44ceb2892f41385ff616 2024-11-20T05:16:29,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/e9f2a2edb69e429abbf2efbd646eab88 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/A/e9f2a2edb69e429abbf2efbd646eab88 2024-11-20T05:16:29,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/89fab725657b4008bc1d33b4d2dbdee9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/89fab725657b4008bc1d33b4d2dbdee9 2024-11-20T05:16:29,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/acee8cc7231d4e1dadf89d4761fe6c80 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/acee8cc7231d4e1dadf89d4761fe6c80 2024-11-20T05:16:29,024 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c842b073bcb443e6ad202b535e1beee9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/B/c842b073bcb443e6ad202b535e1beee9 2024-11-20T05:16:29,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0a3eae17aa6f47c0abdefdf7a2ccfc16 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/0a3eae17aa6f47c0abdefdf7a2ccfc16 2024-11-20T05:16:29,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/72f09ef8878147bc984a69baadcd99bd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/72f09ef8878147bc984a69baadcd99bd 2024-11-20T05:16:29,027 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/f89b87c7d958405dab26393212394f8c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/C/f89b87c7d958405dab26393212394f8c 2024-11-20T05:16:29,029 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/recovered.edits/307.seqid to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6/recovered.edits/307.seqid 2024-11-20T05:16:29,029 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,029 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T05:16:29,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T05:16:29,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T05:16:29,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200386dc9d6b0e4234bf58065bb66eecdc_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200386dc9d6b0e4234bf58065bb66eecdc_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027fbb3efac3e471c84a48c58d7fc7ccf_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027fbb3efac3e471c84a48c58d7fc7ccf_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202e3be52baba24d73990e49fc1134840d_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202e3be52baba24d73990e49fc1134840d_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f203787bd1b46d793dfc36a7b1a9bdd_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f203787bd1b46d793dfc36a7b1a9bdd_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112044395dd07576480e8bdb430b7973a103_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112044395dd07576480e8bdb430b7973a103_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,040 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204d9904d30f2a4d9a88e37bddd498aa14_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204d9904d30f2a4d9a88e37bddd498aa14_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112054811b19297c4b38909e6ec06bf1ec38_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112054811b19297c4b38909e6ec06bf1ec38_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,043 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206450646ec6674af7bab794d0a4955c78_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206450646ec6674af7bab794d0a4955c78_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,044 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112067f3219f3f4e408dad42ef0c6db047a9_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112067f3219f3f4e408dad42ef0c6db047a9_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,045 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120adb5ed794e5f4298a7dbddfcd6041797_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120adb5ed794e5f4298a7dbddfcd6041797_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,046 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b76e141b5bd846bc887bd675dd7be8e6_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b76e141b5bd846bc887bd675dd7be8e6_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,048 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdbc69521dcf47b3a1f4ff7605f42f4c_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdbc69521dcf47b3a1f4ff7605f42f4c_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,049 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cfd22de0596748fa97c926886919ece1_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cfd22de0596748fa97c926886919ece1_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,049 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3db1d4ade1a46a9a65752718e5d02c2_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3db1d4ade1a46a9a65752718e5d02c2_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,050 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0edac34261b48abb7f03546fdd08d59_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0edac34261b48abb7f03546fdd08d59_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,051 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f987add547914ff8bc17dd715ae174d1_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f987add547914ff8bc17dd715ae174d1_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,052 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fe45c53c8ef644bbaf40c41f4f5fc4be_e74402c4b2528f25097db0ff071d71d6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fe45c53c8ef644bbaf40c41f4f5fc4be_e74402c4b2528f25097db0ff071d71d6 2024-11-20T05:16:29,052 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T05:16:29,054 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:29,055 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T05:16:29,057 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T05:16:29,058 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:29,058 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T05:16:29,058 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732079789058"}]},"ts":"9223372036854775807"} 2024-11-20T05:16:29,059 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T05:16:29,059 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e74402c4b2528f25097db0ff071d71d6, NAME => 'TestAcidGuarantees,,1732079756042.e74402c4b2528f25097db0ff071d71d6.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T05:16:29,059 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T05:16:29,060 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732079789059"}]},"ts":"9223372036854775807"} 2024-11-20T05:16:29,061 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T05:16:29,061 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:29,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-20T05:16:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T05:16:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T05:16:29,118 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:29,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 250), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=4583 (was 5358) 2024-11-20T05:16:29,138 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=4583 2024-11-20T05:16:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:16:29,139 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:16:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:29,142 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T05:16:29,142 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:29,142 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-20T05:16:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-20T05:16:29,143 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T05:16:29,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742232_1408 (size=1061) 2024-11-20T05:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-20T05:16:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-20T05:16:29,551 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:16:29,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742233_1409 (size=53) 2024-11-20T05:16:29,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:16:29,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bef9917e0871717b985dc9ca0a53e5ad, disabling compactions & flushes 2024-11-20T05:16:29,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. after waiting 0 ms 2024-11-20T05:16:29,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bef9917e0871717b985dc9ca0a53e5ad: Waiting for close lock at 1732079789572Disabling compacts and flushes for region at 1732079789572Disabling writes for close at 1732079789572Writing region close event to WAL at 1732079789572Closed at 1732079789572 2024-11-20T05:16:29,573 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T05:16:29,574 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732079789573"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732079789573"}]},"ts":"1732079789573"} 2024-11-20T05:16:29,577 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T05:16:29,578 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T05:16:29,578 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079789578"}]},"ts":"1732079789578"} 2024-11-20T05:16:29,580 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T05:16:29,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, ASSIGN}] 2024-11-20T05:16:29,582 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, ASSIGN 2024-11-20T05:16:29,583 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, ASSIGN; state=OFFLINE, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=false 2024-11-20T05:16:29,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=bef9917e0871717b985dc9ca0a53e5ad, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:29,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, ASSIGN because future has completed 2024-11-20T05:16:29,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:16:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-20T05:16:29,894 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,894 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:16:29,896 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,896 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:16:29,896 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,896 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,898 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,900 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:29,900 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bef9917e0871717b985dc9ca0a53e5ad columnFamilyName A 2024-11-20T05:16:29,900 DEBUG [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:29,901 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.HStore(327): Store=bef9917e0871717b985dc9ca0a53e5ad/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:29,901 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,903 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:29,903 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bef9917e0871717b985dc9ca0a53e5ad columnFamilyName B 2024-11-20T05:16:29,903 DEBUG [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:29,904 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.HStore(327): Store=bef9917e0871717b985dc9ca0a53e5ad/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:29,904 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,906 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:29,906 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bef9917e0871717b985dc9ca0a53e5ad columnFamilyName C 2024-11-20T05:16:29,906 DEBUG [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:29,907 INFO [StoreOpener-bef9917e0871717b985dc9ca0a53e5ad-1 {}] regionserver.HStore(327): Store=bef9917e0871717b985dc9ca0a53e5ad/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:29,907 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,907 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,908 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,909 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,911 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,911 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,912 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:16:29,912 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,914 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:16:29,914 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened bef9917e0871717b985dc9ca0a53e5ad; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62413926, jitterRate=-0.06996002793312073}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:16:29,914 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:29,915 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for bef9917e0871717b985dc9ca0a53e5ad: Running coprocessor pre-open hook at 1732079789896Writing region info on filesystem at 1732079789896Initializing all the Stores at 1732079789897 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079789897Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079789898 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079789898Cleaning up temporary data from old regions at 1732079789911 (+13 ms)Running coprocessor post-open hooks at 1732079789914 (+3 ms)Region opened successfully at 1732079789915 (+1 ms) 2024-11-20T05:16:29,915 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., pid=112, masterSystemTime=1732079789887 2024-11-20T05:16:29,917 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,917 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:29,917 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=bef9917e0871717b985dc9ca0a53e5ad, regionState=OPEN, openSeqNum=2, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:29,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:16:29,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-20T05:16:29,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 in 184 msec 2024-11-20T05:16:29,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-20T05:16:29,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, ASSIGN in 340 msec 2024-11-20T05:16:29,922 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T05:16:29,922 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079789922"}]},"ts":"1732079789922"} 2024-11-20T05:16:29,923 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T05:16:29,924 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T05:16:29,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 785 msec 2024-11-20T05:16:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-20T05:16:30,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:30,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54872992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,280 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a8fbc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,282 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2953059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,283 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,285 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T05:16:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 51d5d2011a1c,37927,1732079654666 2024-11-20T05:16:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d0b2c31 2024-11-20T05:16:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T05:16:30,287 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T05:16:30,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514e87d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,289 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3875a4ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,290 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63dadda4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250ddf0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2811bfa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545ccc31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc2be37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,296 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b32d8b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d1ec4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c47751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,301 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ecd8a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d494775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e1154a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b69e88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,307 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8919b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6735, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,310 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d74ef3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,311 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39611fd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19aa15e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,313 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e6c6b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,315 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b05e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f3cd49e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,316 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5d768b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ddff570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b78dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28579ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,320 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0c161c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c5ed93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:30,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:30,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e039499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:30,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:30,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b614134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:30,323 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,323 DEBUG [Thread-1872 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,323 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,323 DEBUG [Thread-1876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,323 DEBUG [Thread-1874 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,323 DEBUG [Thread-1857 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,324 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,324 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,324 DEBUG [Thread-1868 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,325 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,325 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,327 DEBUG [Thread-1874 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:30,327 DEBUG [Thread-1874 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-20T05:16:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T05:16:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2] 2024-11-20T05:16:30,332 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:30,333 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:30,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:30,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:16:30,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:30,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:30,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:30,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:30,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:30,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:30,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/441ed964d9b44d0fa43b309ebfec30f9 is 50, key is test_row_0/A:col10/1732079790332/Put/seqid=0 2024-11-20T05:16:30,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742234_1410 (size=14341) 2024-11-20T05:16:30,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079850373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732079790359, max timestamp=1732079790375Num uniques 859; 2024-11-20T05:16:30,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T05:16:30,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079790362, max timestamp=1732079790375Num uniques 859; 2024-11-20T05:16:30,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:16:30,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079790362, max timestamp=1732079790375Num uniques 859; 2024-11-20T05:16:30,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:16:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079850408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079850410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079850416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079850418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T05:16:30,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T05:16:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:30,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079850487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079850527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079850527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079850527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079850537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T05:16:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:30,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T05:16:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079850707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079850737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079850737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079850738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079850757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:30,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/441ed964d9b44d0fa43b309ebfec30f9 2024-11-20T05:16:30,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T05:16:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:30,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/7f66696bee3342d09416ef14d6da0c86 is 50, key is test_row_0/B:col10/1732079790332/Put/seqid=0 2024-11-20T05:16:30,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:30,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742235_1411 (size=12001) 2024-11-20T05:16:30,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/7f66696bee3342d09416ef14d6da0c86 2024-11-20T05:16:30,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/69d688ef589c47afa18edd13208bedb1 is 50, key is test_row_0/C:col10/1732079790332/Put/seqid=0 2024-11-20T05:16:30,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742236_1412 (size=12001) 2024-11-20T05:16:30,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/69d688ef589c47afa18edd13208bedb1 2024-11-20T05:16:30,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/441ed964d9b44d0fa43b309ebfec30f9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/441ed964d9b44d0fa43b309ebfec30f9 2024-11-20T05:16:30,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/441ed964d9b44d0fa43b309ebfec30f9, entries=200, sequenceid=11, filesize=14.0 K 2024-11-20T05:16:30,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/7f66696bee3342d09416ef14d6da0c86 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7f66696bee3342d09416ef14d6da0c86 2024-11-20T05:16:30,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7f66696bee3342d09416ef14d6da0c86, entries=150, sequenceid=11, filesize=11.7 K 2024-11-20T05:16:30,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/69d688ef589c47afa18edd13208bedb1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/69d688ef589c47afa18edd13208bedb1 2024-11-20T05:16:30,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/69d688ef589c47afa18edd13208bedb1, entries=150, sequenceid=11, filesize=11.7 K 2024-11-20T05:16:30,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bef9917e0871717b985dc9ca0a53e5ad in 500ms, sequenceid=11, compaction requested=false 2024-11-20T05:16:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:30,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T05:16:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:30,945 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T05:16:30,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:30,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:30,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:30,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:30,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:30,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/1fa1a90d8711443892eb4d772568b661 is 50, key is test_row_0/A:col10/1732079790391/Put/seqid=0 2024-11-20T05:16:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742237_1413 (size=12001) 2024-11-20T05:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T05:16:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:31,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079851047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079851048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079851054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079851055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079851077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079851167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079851168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079851168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,354 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/1fa1a90d8711443892eb4d772568b661 2024-11-20T05:16:31,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f783077d7d9d4361b5b410ad6b63db87 is 50, key is test_row_0/B:col10/1732079790391/Put/seqid=0 2024-11-20T05:16:31,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742238_1414 (size=12001) 2024-11-20T05:16:31,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079851377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079851377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079851387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T05:16:31,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079851567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079851587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079851687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079851687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079851707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:31,764 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f783077d7d9d4361b5b410ad6b63db87 2024-11-20T05:16:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/a621d596c7914220ab7b080f62d1f3dc is 50, key is test_row_0/C:col10/1732079790391/Put/seqid=0 2024-11-20T05:16:31,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742239_1415 (size=12001) 2024-11-20T05:16:32,173 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/a621d596c7914220ab7b080f62d1f3dc 2024-11-20T05:16:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/1fa1a90d8711443892eb4d772568b661 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/1fa1a90d8711443892eb4d772568b661 2024-11-20T05:16:32,180 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/1fa1a90d8711443892eb4d772568b661, entries=150, sequenceid=47, filesize=11.7 K 2024-11-20T05:16:32,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f783077d7d9d4361b5b410ad6b63db87 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f783077d7d9d4361b5b410ad6b63db87 2024-11-20T05:16:32,184 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f783077d7d9d4361b5b410ad6b63db87, entries=150, sequenceid=47, filesize=11.7 K 2024-11-20T05:16:32,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/a621d596c7914220ab7b080f62d1f3dc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a621d596c7914220ab7b080f62d1f3dc 2024-11-20T05:16:32,189 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a621d596c7914220ab7b080f62d1f3dc, entries=150, sequenceid=47, filesize=11.7 K 2024-11-20T05:16:32,189 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for bef9917e0871717b985dc9ca0a53e5ad in 1244ms, sequenceid=47, compaction requested=false 2024-11-20T05:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-20T05:16:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-20T05:16:32,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T05:16:32,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-11-20T05:16:32,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8670 sec 2024-11-20T05:16:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:32,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T05:16:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:32,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/0870ef5ebf9846c08d5db6f92c1cf0ad is 50, key is test_row_0/A:col10/1732079792199/Put/seqid=0 2024-11-20T05:16:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742240_1416 (size=16681) 2024-11-20T05:16:32,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079852253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079852260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079852262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079852367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079852367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079852377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T05:16:32,478 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:32,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-20T05:16:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T05:16:32,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:32,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:32,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T05:16:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079852577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079852577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079852597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079852597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079852597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,604 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,604 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/0870ef5ebf9846c08d5db6f92c1cf0ad 2024-11-20T05:16:32,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/7890ac525d9d448487de89a992ebb389 is 50, key is test_row_0/B:col10/1732079792199/Put/seqid=0 2024-11-20T05:16:32,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742241_1417 (size=12001) 2024-11-20T05:16:32,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/7890ac525d9d448487de89a992ebb389 2024-11-20T05:16:32,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T05:16:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:32,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:32,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/f223058744964e0ca1458b28d3e5ce5c is 50, key is test_row_0/C:col10/1732079792199/Put/seqid=0 2024-11-20T05:16:32,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742242_1418 (size=12001) 2024-11-20T05:16:32,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/f223058744964e0ca1458b28d3e5ce5c 2024-11-20T05:16:32,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/0870ef5ebf9846c08d5db6f92c1cf0ad as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0870ef5ebf9846c08d5db6f92c1cf0ad 2024-11-20T05:16:32,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0870ef5ebf9846c08d5db6f92c1cf0ad, entries=250, sequenceid=59, filesize=16.3 K 2024-11-20T05:16:32,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/7890ac525d9d448487de89a992ebb389 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7890ac525d9d448487de89a992ebb389 2024-11-20T05:16:32,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7890ac525d9d448487de89a992ebb389, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T05:16:32,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/f223058744964e0ca1458b28d3e5ce5c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/f223058744964e0ca1458b28d3e5ce5c 2024-11-20T05:16:32,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/f223058744964e0ca1458b28d3e5ce5c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T05:16:32,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bef9917e0871717b985dc9ca0a53e5ad in 475ms, sequenceid=59, compaction requested=true 2024-11-20T05:16:32,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:32,675 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:32,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:32,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:32,675 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:32,679 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:32,679 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:32,679 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:32,679 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:32,679 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:32,679 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:32,679 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7f66696bee3342d09416ef14d6da0c86, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f783077d7d9d4361b5b410ad6b63db87, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7890ac525d9d448487de89a992ebb389] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.2 K 2024-11-20T05:16:32,679 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/441ed964d9b44d0fa43b309ebfec30f9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/1fa1a90d8711443892eb4d772568b661, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0870ef5ebf9846c08d5db6f92c1cf0ad] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=42.0 K 2024-11-20T05:16:32,680 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f66696bee3342d09416ef14d6da0c86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732079790332 2024-11-20T05:16:32,680 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 441ed964d9b44d0fa43b309ebfec30f9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732079790330 2024-11-20T05:16:32,680 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f783077d7d9d4361b5b410ad6b63db87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732079790391 2024-11-20T05:16:32,680 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa1a90d8711443892eb4d772568b661, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732079790391 2024-11-20T05:16:32,680 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7890ac525d9d448487de89a992ebb389, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732079791047 2024-11-20T05:16:32,681 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0870ef5ebf9846c08d5db6f92c1cf0ad, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732079791047 2024-11-20T05:16:32,702 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:32,703 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/8e43c1af460d41089514c04cb295ad99 is 50, key is test_row_0/B:col10/1732079792199/Put/seqid=0 2024-11-20T05:16:32,706 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:32,706 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/85aa61c2b1ac4615b047b3ace5927006 is 50, key is test_row_0/A:col10/1732079792199/Put/seqid=0 2024-11-20T05:16:32,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742243_1419 (size=12104) 2024-11-20T05:16:32,725 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/85aa61c2b1ac4615b047b3ace5927006 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/85aa61c2b1ac4615b047b3ace5927006 2024-11-20T05:16:32,730 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into 85aa61c2b1ac4615b047b3ace5927006(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:32,730 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:32,730 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=13, startTime=1732079792675; duration=0sec 2024-11-20T05:16:32,730 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:32,730 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:32,730 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:32,733 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:32,733 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:32,733 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:32,733 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/69d688ef589c47afa18edd13208bedb1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a621d596c7914220ab7b080f62d1f3dc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/f223058744964e0ca1458b28d3e5ce5c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.2 K 2024-11-20T05:16:32,733 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d688ef589c47afa18edd13208bedb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732079790332 2024-11-20T05:16:32,734 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a621d596c7914220ab7b080f62d1f3dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732079790391 2024-11-20T05:16:32,734 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f223058744964e0ca1458b28d3e5ce5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732079791047 2024-11-20T05:16:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742244_1420 (size=12104) 2024-11-20T05:16:32,746 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/8e43c1af460d41089514c04cb295ad99 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8e43c1af460d41089514c04cb295ad99 2024-11-20T05:16:32,750 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:32,751 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/b1284c81b46f4a5a8f8df32542c7d0c1 is 50, key is test_row_0/C:col10/1732079792199/Put/seqid=0 2024-11-20T05:16:32,755 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into 8e43c1af460d41089514c04cb295ad99(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:32,755 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:32,755 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=13, startTime=1732079792675; duration=0sec 2024-11-20T05:16:32,755 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:32,755 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:32,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742245_1421 (size=12104) 2024-11-20T05:16:32,781 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/b1284c81b46f4a5a8f8df32542c7d0c1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b1284c81b46f4a5a8f8df32542c7d0c1 2024-11-20T05:16:32,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T05:16:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:32,788 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T05:16:32,788 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into b1284c81b46f4a5a8f8df32542c7d0c1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:32,788 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:32,788 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=13, startTime=1732079792676; duration=0sec 2024-11-20T05:16:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:32,788 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:32,788 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:32,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/0a556276deba4ea39912a5182dc8fc14 is 50, key is test_row_1/A:col10/1732079792260/Put/seqid=0 2024-11-20T05:16:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T05:16:32,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742246_1422 (size=9657) 2024-11-20T05:16:32,813 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/0a556276deba4ea39912a5182dc8fc14 2024-11-20T05:16:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/8754ee2ae9124fe4b4e9d534a01d1e04 is 50, key is test_row_1/B:col10/1732079792260/Put/seqid=0 2024-11-20T05:16:32,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742247_1423 (size=9657) 2024-11-20T05:16:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:32,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:32,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079852922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079852923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079852931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079853037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079853047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079853047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T05:16:33,249 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/8754ee2ae9124fe4b4e9d534a01d1e04 2024-11-20T05:16:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079853247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/61e6893e5a824bd2a0e281f4b387028b is 50, key is test_row_1/C:col10/1732079792260/Put/seqid=0 2024-11-20T05:16:33,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079853271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079853271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742248_1424 (size=9657) 2024-11-20T05:16:33,293 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/61e6893e5a824bd2a0e281f4b387028b 2024-11-20T05:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/0a556276deba4ea39912a5182dc8fc14 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0a556276deba4ea39912a5182dc8fc14 2024-11-20T05:16:33,299 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0a556276deba4ea39912a5182dc8fc14, entries=100, sequenceid=82, filesize=9.4 K 2024-11-20T05:16:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/8754ee2ae9124fe4b4e9d534a01d1e04 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8754ee2ae9124fe4b4e9d534a01d1e04 2024-11-20T05:16:33,305 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8754ee2ae9124fe4b4e9d534a01d1e04, entries=100, sequenceid=82, filesize=9.4 K 2024-11-20T05:16:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/61e6893e5a824bd2a0e281f4b387028b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/61e6893e5a824bd2a0e281f4b387028b 2024-11-20T05:16:33,313 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/61e6893e5a824bd2a0e281f4b387028b, entries=100, sequenceid=82, filesize=9.4 K 2024-11-20T05:16:33,313 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for bef9917e0871717b985dc9ca0a53e5ad in 525ms, sequenceid=82, compaction requested=false 2024-11-20T05:16:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-20T05:16:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-20T05:16:33,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-20T05:16:33,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 835 msec 2024-11-20T05:16:33,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 843 msec 2024-11-20T05:16:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:33,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T05:16:33,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:33,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:33,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:33,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:33,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:33,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:33,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/354e707e7411457d83a0fed8ed6b9f33 is 50, key is test_row_0/A:col10/1732079793570/Put/seqid=0 2024-11-20T05:16:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742249_1425 (size=14341) 2024-11-20T05:16:33,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/354e707e7411457d83a0fed8ed6b9f33 2024-11-20T05:16:33,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/2f0aea35bf1e43d79aaa7eefbc326efb is 50, key is test_row_0/B:col10/1732079793570/Put/seqid=0 2024-11-20T05:16:33,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742250_1426 (size=12001) 2024-11-20T05:16:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T05:16:33,617 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:33,618 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-20T05:16:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T05:16:33,619 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:33,619 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:33,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:33,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079853630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079853637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079853641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T05:16:33,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079853747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079853747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079853757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T05:16:33,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:33,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:33,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:33,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:33,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:33,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T05:16:33,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:33,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:33,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:33,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:33,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T05:16:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079853957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079853958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079853977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:33,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/2f0aea35bf1e43d79aaa7eefbc326efb 2024-11-20T05:16:33,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/bd2982d1dcec401c964b7274fb14b5e7 is 50, key is test_row_0/C:col10/1732079793570/Put/seqid=0 2024-11-20T05:16:33,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742251_1427 (size=12001) 2024-11-20T05:16:34,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T05:16:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T05:16:34,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:34,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T05:16:34,258 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T05:16:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079854277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079854277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079854287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T05:16:34,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:34,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:34,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/bd2982d1dcec401c964b7274fb14b5e7 2024-11-20T05:16:34,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/354e707e7411457d83a0fed8ed6b9f33 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/354e707e7411457d83a0fed8ed6b9f33 2024-11-20T05:16:34,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/354e707e7411457d83a0fed8ed6b9f33, entries=200, sequenceid=95, filesize=14.0 K 2024-11-20T05:16:34,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/2f0aea35bf1e43d79aaa7eefbc326efb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2f0aea35bf1e43d79aaa7eefbc326efb 2024-11-20T05:16:34,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2f0aea35bf1e43d79aaa7eefbc326efb, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T05:16:34,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/bd2982d1dcec401c964b7274fb14b5e7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd2982d1dcec401c964b7274fb14b5e7 2024-11-20T05:16:34,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd2982d1dcec401c964b7274fb14b5e7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T05:16:34,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for bef9917e0871717b985dc9ca0a53e5ad in 849ms, sequenceid=95, compaction requested=true 2024-11-20T05:16:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:34,421 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:34,421 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:34,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:34,422 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:34,422 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:34,422 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:34,422 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,422 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:34,422 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,422 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/85aa61c2b1ac4615b047b3ace5927006, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0a556276deba4ea39912a5182dc8fc14, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/354e707e7411457d83a0fed8ed6b9f33] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.3 K 2024-11-20T05:16:34,422 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8e43c1af460d41089514c04cb295ad99, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8754ee2ae9124fe4b4e9d534a01d1e04, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2f0aea35bf1e43d79aaa7eefbc326efb] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=33.0 K 2024-11-20T05:16:34,422 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85aa61c2b1ac4615b047b3ace5927006, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732079791047 2024-11-20T05:16:34,422 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e43c1af460d41089514c04cb295ad99, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732079791047 2024-11-20T05:16:34,423 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a556276deba4ea39912a5182dc8fc14, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732079792252 2024-11-20T05:16:34,423 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8754ee2ae9124fe4b4e9d534a01d1e04, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732079792252 2024-11-20T05:16:34,423 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 354e707e7411457d83a0fed8ed6b9f33, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079792908 2024-11-20T05:16:34,423 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f0aea35bf1e43d79aaa7eefbc326efb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079792918 2024-11-20T05:16:34,430 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:34,431 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/de0ae086cefb42bb85b7ca7a9b634857 is 50, key is test_row_0/A:col10/1732079793570/Put/seqid=0 2024-11-20T05:16:34,433 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:34,434 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/499f23f98e224f8e8c9a4cbc669923a4 is 50, key is test_row_0/B:col10/1732079793570/Put/seqid=0 2024-11-20T05:16:34,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742253_1429 (size=12207) 2024-11-20T05:16:34,449 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/499f23f98e224f8e8c9a4cbc669923a4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/499f23f98e224f8e8c9a4cbc669923a4 2024-11-20T05:16:34,458 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into 499f23f98e224f8e8c9a4cbc669923a4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:34,458 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:34,458 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=13, startTime=1732079794421; duration=0sec 2024-11-20T05:16:34,458 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:34,458 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:34,458 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:34,459 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:34,459 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:34,459 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,459 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b1284c81b46f4a5a8f8df32542c7d0c1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/61e6893e5a824bd2a0e281f4b387028b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd2982d1dcec401c964b7274fb14b5e7] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=33.0 K 2024-11-20T05:16:34,460 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b1284c81b46f4a5a8f8df32542c7d0c1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732079791047 2024-11-20T05:16:34,460 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e6893e5a824bd2a0e281f4b387028b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732079792252 2024-11-20T05:16:34,460 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2982d1dcec401c964b7274fb14b5e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079792918 2024-11-20T05:16:34,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742252_1428 (size=12207) 2024-11-20T05:16:34,469 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:34,469 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/8266b8f7c1ad4507869990f5507c8eb5 is 50, key is test_row_0/C:col10/1732079793570/Put/seqid=0 2024-11-20T05:16:34,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742254_1430 (size=12207) 2024-11-20T05:16:34,481 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/8266b8f7c1ad4507869990f5507c8eb5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/8266b8f7c1ad4507869990f5507c8eb5 2024-11-20T05:16:34,486 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into 8266b8f7c1ad4507869990f5507c8eb5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:34,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:34,486 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=13, startTime=1732079794421; duration=0sec 2024-11-20T05:16:34,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:34,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:34,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T05:16:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:34,535 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T05:16:34,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:34,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:34,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:34,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:34,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:34,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:34,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ff1e743dbd134c73937eec434278d635 is 50, key is test_row_0/A:col10/1732079793635/Put/seqid=0 2024-11-20T05:16:34,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742255_1431 (size=12001) 2024-11-20T05:16:34,549 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ff1e743dbd134c73937eec434278d635 2024-11-20T05:16:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/08ae19ead04f40449896c662b381a7f1 is 50, key is test_row_0/B:col10/1732079793635/Put/seqid=0 2024-11-20T05:16:34,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742256_1432 (size=12001) 2024-11-20T05:16:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:34,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:34,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079854658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079854659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T05:16:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079854767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079854777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079854797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079854797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079854798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,869 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/de0ae086cefb42bb85b7ca7a9b634857 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/de0ae086cefb42bb85b7ca7a9b634857 2024-11-20T05:16:34,873 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into de0ae086cefb42bb85b7ca7a9b634857(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:34,873 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:34,873 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=13, startTime=1732079794421; duration=0sec 2024-11-20T05:16:34,873 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:34,873 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:34,958 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/08ae19ead04f40449896c662b381a7f1 2024-11-20T05:16:34,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/fef48022bf33480e97caa619a58d4983 is 50, key is test_row_0/C:col10/1732079793635/Put/seqid=0 2024-11-20T05:16:34,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742257_1433 (size=12001) 2024-11-20T05:16:34,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079854987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079854987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079855307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079855308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,387 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/fef48022bf33480e97caa619a58d4983 2024-11-20T05:16:35,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ff1e743dbd134c73937eec434278d635 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ff1e743dbd134c73937eec434278d635 2024-11-20T05:16:35,395 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ff1e743dbd134c73937eec434278d635, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T05:16:35,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/08ae19ead04f40449896c662b381a7f1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/08ae19ead04f40449896c662b381a7f1 2024-11-20T05:16:35,400 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/08ae19ead04f40449896c662b381a7f1, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T05:16:35,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/fef48022bf33480e97caa619a58d4983 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/fef48022bf33480e97caa619a58d4983 2024-11-20T05:16:35,406 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/fef48022bf33480e97caa619a58d4983, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T05:16:35,407 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for bef9917e0871717b985dc9ca0a53e5ad in 871ms, sequenceid=116, compaction requested=false 2024-11-20T05:16:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-20T05:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-20T05:16:35,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-20T05:16:35,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7890 sec 2024-11-20T05:16:35,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.7930 sec 2024-11-20T05:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T05:16:35,767 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:35,767 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-20T05:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T05:16:35,769 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:35,769 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:35,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:35,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T05:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:35,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/e0d097d645944e76a11204ab36b49d51 is 50, key is test_row_0/A:col10/1732079795823/Put/seqid=0 2024-11-20T05:16:35,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742258_1434 (size=16781) 2024-11-20T05:16:35,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/e0d097d645944e76a11204ab36b49d51 2024-11-20T05:16:35,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/4715fbed395b43a68ce6d02fcee8ace2 is 50, key is test_row_0/B:col10/1732079795823/Put/seqid=0 2024-11-20T05:16:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079855852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079855853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079855855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079855859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742259_1435 (size=12101) 2024-11-20T05:16:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/4715fbed395b43a68ce6d02fcee8ace2 2024-11-20T05:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T05:16:35,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079855866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/9236a7358fd444098e0e9bab2f366bd5 is 50, key is test_row_0/C:col10/1732079795823/Put/seqid=0 2024-11-20T05:16:35,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742260_1436 (size=12101) 2024-11-20T05:16:35,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T05:16:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:35,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079855978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079855978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079855978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079855978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079855987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T05:16:36,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:36,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:36,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T05:16:36,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079856197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079856197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079856198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079856198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079856198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T05:16:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:36,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/9236a7358fd444098e0e9bab2f366bd5 2024-11-20T05:16:36,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/e0d097d645944e76a11204ab36b49d51 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/e0d097d645944e76a11204ab36b49d51 2024-11-20T05:16:36,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/e0d097d645944e76a11204ab36b49d51, entries=250, sequenceid=132, filesize=16.4 K 2024-11-20T05:16:36,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/4715fbed395b43a68ce6d02fcee8ace2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4715fbed395b43a68ce6d02fcee8ace2 2024-11-20T05:16:36,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4715fbed395b43a68ce6d02fcee8ace2, entries=150, sequenceid=132, filesize=11.8 K 2024-11-20T05:16:36,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/9236a7358fd444098e0e9bab2f366bd5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9236a7358fd444098e0e9bab2f366bd5 2024-11-20T05:16:36,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9236a7358fd444098e0e9bab2f366bd5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-20T05:16:36,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for bef9917e0871717b985dc9ca0a53e5ad in 517ms, sequenceid=132, compaction requested=true 2024-11-20T05:16:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:36,343 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T05:16:36,343 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:36,344 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:36,344 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:36,344 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,344 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/de0ae086cefb42bb85b7ca7a9b634857, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ff1e743dbd134c73937eec434278d635, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/e0d097d645944e76a11204ab36b49d51] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=40.0 K 2024-11-20T05:16:36,344 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:36,345 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:36,345 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,345 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/499f23f98e224f8e8c9a4cbc669923a4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/08ae19ead04f40449896c662b381a7f1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4715fbed395b43a68ce6d02fcee8ace2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.5 K 2024-11-20T05:16:36,345 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting de0ae086cefb42bb85b7ca7a9b634857, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079792918 2024-11-20T05:16:36,345 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 499f23f98e224f8e8c9a4cbc669923a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079792918 2024-11-20T05:16:36,345 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1e743dbd134c73937eec434278d635, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732079793622 2024-11-20T05:16:36,348 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ae19ead04f40449896c662b381a7f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732079793622 2024-11-20T05:16:36,348 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d097d645944e76a11204ab36b49d51, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079794643 2024-11-20T05:16:36,349 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4715fbed395b43a68ce6d02fcee8ace2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079794643 2024-11-20T05:16:36,377 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#365 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:36,378 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/95da6be115b04e5a9fd991e305752abb is 50, key is test_row_0/A:col10/1732079795823/Put/seqid=0 2024-11-20T05:16:36,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T05:16:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,383 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:36,386 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:36,387 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/e92aa2d5f4cb4b24ab6599ef7dee58c5 is 50, key is test_row_0/B:col10/1732079795823/Put/seqid=0 2024-11-20T05:16:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T05:16:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/75cc31452eef42a297cb32f87cfc72b7 is 50, key is test_row_0/A:col10/1732079795857/Put/seqid=0 2024-11-20T05:16:36,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742262_1438 (size=12409) 2024-11-20T05:16:36,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742261_1437 (size=12409) 2024-11-20T05:16:36,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742263_1439 (size=12151) 2024-11-20T05:16:36,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079856537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079856539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079856539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079856541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079856543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079856657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079856657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079856658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079856658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079856667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,819 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/e92aa2d5f4cb4b24ab6599ef7dee58c5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/e92aa2d5f4cb4b24ab6599ef7dee58c5 2024-11-20T05:16:36,823 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into e92aa2d5f4cb4b24ab6599ef7dee58c5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:36,823 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:36,823 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=13, startTime=1732079796343; duration=0sec 2024-11-20T05:16:36,823 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:36,823 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:36,823 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:36,824 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:36,824 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:36,824 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,824 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/8266b8f7c1ad4507869990f5507c8eb5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/fef48022bf33480e97caa619a58d4983, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9236a7358fd444098e0e9bab2f366bd5] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.5 K 2024-11-20T05:16:36,826 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8266b8f7c1ad4507869990f5507c8eb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732079792918 2024-11-20T05:16:36,827 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting fef48022bf33480e97caa619a58d4983, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732079793622 2024-11-20T05:16:36,828 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9236a7358fd444098e0e9bab2f366bd5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079794643 2024-11-20T05:16:36,830 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/95da6be115b04e5a9fd991e305752abb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/95da6be115b04e5a9fd991e305752abb 2024-11-20T05:16:36,835 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:36,836 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/87f0db5f794f4c90996b789f53608dcb is 50, key is test_row_0/C:col10/1732079795823/Put/seqid=0 2024-11-20T05:16:36,836 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into 95da6be115b04e5a9fd991e305752abb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:36,836 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:36,836 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=13, startTime=1732079796343; duration=0sec 2024-11-20T05:16:36,837 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:36,837 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:36,838 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/75cc31452eef42a297cb32f87cfc72b7 2024-11-20T05:16:36,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742264_1440 (size=12409) 2024-11-20T05:16:36,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5945fb08fb96428386ad58e842b3745b is 50, key is test_row_0/B:col10/1732079795857/Put/seqid=0 2024-11-20T05:16:36,849 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/87f0db5f794f4c90996b789f53608dcb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/87f0db5f794f4c90996b789f53608dcb 2024-11-20T05:16:36,854 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into 87f0db5f794f4c90996b789f53608dcb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:36,854 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:36,854 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=13, startTime=1732079796343; duration=0sec 2024-11-20T05:16:36,855 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:36,855 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:36,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742265_1441 (size=12151) 2024-11-20T05:16:36,870 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5945fb08fb96428386ad58e842b3745b 2024-11-20T05:16:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079856868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079856868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079856868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/822bb76a18474bbeb6496027c4327490 is 50, key is test_row_0/C:col10/1732079795857/Put/seqid=0 2024-11-20T05:16:36,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742266_1442 (size=12151) 2024-11-20T05:16:36,884 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/822bb76a18474bbeb6496027c4327490 2024-11-20T05:16:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079856878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079856887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:36,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:36,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:36,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/75cc31452eef42a297cb32f87cfc72b7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/75cc31452eef42a297cb32f87cfc72b7 2024-11-20T05:16:36,899 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/75cc31452eef42a297cb32f87cfc72b7, entries=150, sequenceid=149, filesize=11.9 K 2024-11-20T05:16:36,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5945fb08fb96428386ad58e842b3745b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5945fb08fb96428386ad58e842b3745b 2024-11-20T05:16:36,904 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5945fb08fb96428386ad58e842b3745b, entries=150, sequenceid=149, filesize=11.9 K 2024-11-20T05:16:36,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/822bb76a18474bbeb6496027c4327490 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/822bb76a18474bbeb6496027c4327490 2024-11-20T05:16:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T05:16:36,908 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/822bb76a18474bbeb6496027c4327490, entries=150, sequenceid=149, filesize=11.9 K 2024-11-20T05:16:36,909 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for bef9917e0871717b985dc9ca0a53e5ad in 526ms, sequenceid=149, compaction requested=false 2024-11-20T05:16:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-20T05:16:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-20T05:16:36,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T05:16:36,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1410 sec 2024-11-20T05:16:36,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1440 sec 2024-11-20T05:16:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:37,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:16:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:37,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa is 50, key is test_row_0/A:col10/1732079797183/Put/seqid=0 2024-11-20T05:16:37,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742267_1443 (size=12151) 2024-11-20T05:16:37,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079857242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079857242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079857243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079857243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079857243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079857357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079857367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079857367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079857367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079857367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079857567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079857577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079857577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa 2024-11-20T05:16:37,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9e844e18447f4ab3b742113d0e993f62 is 50, key is test_row_0/B:col10/1732079797183/Put/seqid=0 2024-11-20T05:16:37,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079857587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079857587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742268_1444 (size=12151) 2024-11-20T05:16:37,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079857887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079857897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079857897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T05:16:37,918 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:37,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079857907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:37,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079857907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-20T05:16:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T05:16:37,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:37,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:37,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:38,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9e844e18447f4ab3b742113d0e993f62 2024-11-20T05:16:38,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/bd13fc502ca3492d966dfb38d8c1ff73 is 50, key is test_row_0/C:col10/1732079797183/Put/seqid=0 2024-11-20T05:16:38,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742269_1445 (size=12151) 2024-11-20T05:16:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T05:16:38,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T05:16:38,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079858397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/bd13fc502ca3492d966dfb38d8c1ff73 2024-11-20T05:16:38,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa 2024-11-20T05:16:38,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T05:16:38,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9e844e18447f4ab3b742113d0e993f62 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9e844e18447f4ab3b742113d0e993f62 2024-11-20T05:16:38,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9e844e18447f4ab3b742113d0e993f62, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T05:16:38,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/bd13fc502ca3492d966dfb38d8c1ff73 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd13fc502ca3492d966dfb38d8c1ff73 2024-11-20T05:16:38,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd13fc502ca3492d966dfb38d8c1ff73, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T05:16:38,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for bef9917e0871717b985dc9ca0a53e5ad in 1245ms, sequenceid=169, compaction requested=true 2024-11-20T05:16:38,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:38,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:38,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:38,429 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:38,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:38,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:38,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:38,429 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:38,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:38,429 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:38,429 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:38,430 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:38,430 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,430 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/95da6be115b04e5a9fd991e305752abb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/75cc31452eef42a297cb32f87cfc72b7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.9 K 2024-11-20T05:16:38,430 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/e92aa2d5f4cb4b24ab6599ef7dee58c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5945fb08fb96428386ad58e842b3745b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9e844e18447f4ab3b742113d0e993f62] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.9 K 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95da6be115b04e5a9fd991e305752abb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079794643 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e92aa2d5f4cb4b24ab6599ef7dee58c5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079794643 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75cc31452eef42a297cb32f87cfc72b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732079795848 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5945fb08fb96428386ad58e842b3745b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732079795848 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d7b7a8e80f24ccc8215ff2a19fc9bfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079796537 2024-11-20T05:16:38,430 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e844e18447f4ab3b742113d0e993f62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079796537 2024-11-20T05:16:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:38,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T05:16:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:38,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:38,437 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:38,438 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/3a2de489747740f09dac3c80bee7f5cd is 50, key is test_row_0/B:col10/1732079797183/Put/seqid=0 2024-11-20T05:16:38,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/13482fecb1ce41edbc4fa5cc7239b18c is 50, key is test_row_0/A:col10/1732079798430/Put/seqid=0 2024-11-20T05:16:38,441 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:38,441 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/637052a1ae0b4a18a96d4567f29388f4 is 50, key is test_row_0/A:col10/1732079797183/Put/seqid=0 2024-11-20T05:16:38,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742270_1446 (size=16931) 2024-11-20T05:16:38,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742271_1447 (size=12561) 2024-11-20T05:16:38,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742272_1448 (size=12561) 2024-11-20T05:16:38,452 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/3a2de489747740f09dac3c80bee7f5cd as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/3a2de489747740f09dac3c80bee7f5cd 2024-11-20T05:16:38,453 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/637052a1ae0b4a18a96d4567f29388f4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/637052a1ae0b4a18a96d4567f29388f4 2024-11-20T05:16:38,456 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into 3a2de489747740f09dac3c80bee7f5cd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:38,456 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=13, startTime=1732079798429; duration=0sec 2024-11-20T05:16:38,456 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into 637052a1ae0b4a18a96d4567f29388f4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:38,456 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=13, startTime=1732079798429; duration=0sec 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:38,456 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:38,457 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:38,457 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,457 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/87f0db5f794f4c90996b789f53608dcb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/822bb76a18474bbeb6496027c4327490, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd13fc502ca3492d966dfb38d8c1ff73] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=35.9 K 2024-11-20T05:16:38,457 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f0db5f794f4c90996b789f53608dcb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732079794643 2024-11-20T05:16:38,457 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 822bb76a18474bbeb6496027c4327490, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732079795848 2024-11-20T05:16:38,457 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bd13fc502ca3492d966dfb38d8c1ff73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079796537 2024-11-20T05:16:38,462 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:38,462 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/0772bd519a344753ac525c77436339d0 is 50, key is test_row_0/C:col10/1732079797183/Put/seqid=0 2024-11-20T05:16:38,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742273_1449 (size=12561) 2024-11-20T05:16:38,468 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/0772bd519a344753ac525c77436339d0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/0772bd519a344753ac525c77436339d0 2024-11-20T05:16:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079858461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,471 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into 0772bd519a344753ac525c77436339d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:38,471 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:38,471 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=13, startTime=1732079798429; duration=0sec 2024-11-20T05:16:38,472 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:38,472 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079858463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079858464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079858464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T05:16:38,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079858577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079858577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079858578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079858587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079858787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079858787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079858787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079858797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:38,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:38,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/13482fecb1ce41edbc4fa5cc7239b18c 2024-11-20T05:16:38,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/ac27ad75cbfc43a08adeefd4091209b0 is 50, key is test_row_0/B:col10/1732079798430/Put/seqid=0 2024-11-20T05:16:38,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742274_1450 (size=12151) 2024-11-20T05:16:38,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:38,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T05:16:39,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079859107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079859107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079859108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079859117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:39,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:39,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/ac27ad75cbfc43a08adeefd4091209b0 2024-11-20T05:16:39,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/dc6aaf74072e490381e1202888bcdd20 is 50, key is test_row_0/C:col10/1732079798430/Put/seqid=0 2024-11-20T05:16:39,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742275_1451 (size=12151) 2024-11-20T05:16:39,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/dc6aaf74072e490381e1202888bcdd20 2024-11-20T05:16:39,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:39,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:39,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/13482fecb1ce41edbc4fa5cc7239b18c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13482fecb1ce41edbc4fa5cc7239b18c 2024-11-20T05:16:39,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13482fecb1ce41edbc4fa5cc7239b18c, entries=250, sequenceid=188, filesize=16.5 K 2024-11-20T05:16:39,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/ac27ad75cbfc43a08adeefd4091209b0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/ac27ad75cbfc43a08adeefd4091209b0 2024-11-20T05:16:39,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/ac27ad75cbfc43a08adeefd4091209b0, entries=150, sequenceid=188, filesize=11.9 K 2024-11-20T05:16:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/dc6aaf74072e490381e1202888bcdd20 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/dc6aaf74072e490381e1202888bcdd20 2024-11-20T05:16:39,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/dc6aaf74072e490381e1202888bcdd20, entries=150, sequenceid=188, filesize=11.9 K 2024-11-20T05:16:39,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for bef9917e0871717b985dc9ca0a53e5ad in 889ms, sequenceid=188, compaction requested=false 2024-11-20T05:16:39,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:39,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T05:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:39,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/f63c5a5f821a4d64935faae70daab0ff is 50, key is test_row_0/A:col10/1732079798462/Put/seqid=0 2024-11-20T05:16:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742276_1452 (size=14541) 2024-11-20T05:16:39,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/f63c5a5f821a4d64935faae70daab0ff 2024-11-20T05:16:39,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/fa05d930e2a54ebbbff104f3e22e799f is 50, key is test_row_0/B:col10/1732079798462/Put/seqid=0 2024-11-20T05:16:39,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742277_1453 (size=12151) 2024-11-20T05:16:39,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:39,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:39,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079859531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079859617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079859627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079859627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079859637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079859647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/fa05d930e2a54ebbbff104f3e22e799f 2024-11-20T05:16:39,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/d64fb91db11648f1a03d7b0d5cdcf223 is 50, key is test_row_0/C:col10/1732079798462/Put/seqid=0 2024-11-20T05:16:39,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742278_1454 (size=12151) 2024-11-20T05:16:39,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079859867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:39,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:39,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:40,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:40,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:40,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:40,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T05:16:40,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079860187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:40,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:40,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:40,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:40,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/d64fb91db11648f1a03d7b0d5cdcf223 2024-11-20T05:16:40,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/f63c5a5f821a4d64935faae70daab0ff as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/f63c5a5f821a4d64935faae70daab0ff 2024-11-20T05:16:40,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/f63c5a5f821a4d64935faae70daab0ff, entries=200, sequenceid=205, filesize=14.2 K 2024-11-20T05:16:40,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/fa05d930e2a54ebbbff104f3e22e799f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/fa05d930e2a54ebbbff104f3e22e799f 2024-11-20T05:16:40,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/fa05d930e2a54ebbbff104f3e22e799f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T05:16:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/d64fb91db11648f1a03d7b0d5cdcf223 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/d64fb91db11648f1a03d7b0d5cdcf223 2024-11-20T05:16:40,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/d64fb91db11648f1a03d7b0d5cdcf223, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T05:16:40,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for bef9917e0871717b985dc9ca0a53e5ad in 845ms, sequenceid=205, compaction requested=true 2024-11-20T05:16:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:40,267 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:40,267 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:40,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:40,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:40,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:40,268 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:40,268 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:40,268 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:40,268 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:40,268 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,268 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,269 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/637052a1ae0b4a18a96d4567f29388f4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13482fecb1ce41edbc4fa5cc7239b18c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/f63c5a5f821a4d64935faae70daab0ff] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=43.0 K 2024-11-20T05:16:40,269 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/3a2de489747740f09dac3c80bee7f5cd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/ac27ad75cbfc43a08adeefd4091209b0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/fa05d930e2a54ebbbff104f3e22e799f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=36.0 K 2024-11-20T05:16:40,269 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a2de489747740f09dac3c80bee7f5cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079796537 2024-11-20T05:16:40,269 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 637052a1ae0b4a18a96d4567f29388f4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079796537 2024-11-20T05:16:40,269 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ac27ad75cbfc43a08adeefd4091209b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732079797242 2024-11-20T05:16:40,269 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13482fecb1ce41edbc4fa5cc7239b18c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732079797224 2024-11-20T05:16:40,269 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting fa05d930e2a54ebbbff104f3e22e799f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732079798462 2024-11-20T05:16:40,270 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f63c5a5f821a4d64935faae70daab0ff, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732079798446 2024-11-20T05:16:40,287 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:40,287 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:40,287 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/fb39df31e41a48bb948bacc68be29e93 is 50, key is test_row_0/A:col10/1732079798462/Put/seqid=0 2024-11-20T05:16:40,287 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f1e8833706224bf7b6d69e1dd1e2ceb1 is 50, key is test_row_0/B:col10/1732079798462/Put/seqid=0 2024-11-20T05:16:40,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742279_1455 (size=12663) 2024-11-20T05:16:40,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742280_1456 (size=12663) 2024-11-20T05:16:40,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T05:16:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,364 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T05:16:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:40,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/13c1a61154bc4ca5ad810c46fd292013 is 50, key is test_row_0/A:col10/1732079799515/Put/seqid=0 2024-11-20T05:16:40,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742281_1457 (size=12151) 2024-11-20T05:16:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:40,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079860670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079860671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079860672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079860681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,704 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f1e8833706224bf7b6d69e1dd1e2ceb1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f1e8833706224bf7b6d69e1dd1e2ceb1 2024-11-20T05:16:40,708 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into f1e8833706224bf7b6d69e1dd1e2ceb1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:40,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:40,708 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=13, startTime=1732079800267; duration=0sec 2024-11-20T05:16:40,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:40,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:40,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:40,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:40,708 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/fb39df31e41a48bb948bacc68be29e93 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/fb39df31e41a48bb948bacc68be29e93 2024-11-20T05:16:40,708 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:40,708 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:40,709 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/0772bd519a344753ac525c77436339d0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/dc6aaf74072e490381e1202888bcdd20, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/d64fb91db11648f1a03d7b0d5cdcf223] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=36.0 K 2024-11-20T05:16:40,709 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0772bd519a344753ac525c77436339d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732079796537 2024-11-20T05:16:40,709 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting dc6aaf74072e490381e1202888bcdd20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732079797242 2024-11-20T05:16:40,709 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d64fb91db11648f1a03d7b0d5cdcf223, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732079798462 2024-11-20T05:16:40,712 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into fb39df31e41a48bb948bacc68be29e93(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:40,712 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:40,712 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=13, startTime=1732079800267; duration=0sec 2024-11-20T05:16:40,712 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:40,712 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:40,715 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:40,716 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/c90d18397ac1441baa77e94e6d32cc32 is 50, key is test_row_0/C:col10/1732079798462/Put/seqid=0 2024-11-20T05:16:40,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079860707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742282_1458 (size=12663) 2024-11-20T05:16:40,773 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/13c1a61154bc4ca5ad810c46fd292013 2024-11-20T05:16:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f2745987e524405888f3b81834a980fb is 50, key is test_row_0/B:col10/1732079799515/Put/seqid=0 2024-11-20T05:16:40,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742283_1459 (size=12151) 2024-11-20T05:16:40,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079860787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079860797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079860797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079860797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079860997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079861007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079861007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079861007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,123 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/c90d18397ac1441baa77e94e6d32cc32 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/c90d18397ac1441baa77e94e6d32cc32 2024-11-20T05:16:41,129 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into c90d18397ac1441baa77e94e6d32cc32(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:41,129 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:41,129 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=13, startTime=1732079800268; duration=0sec 2024-11-20T05:16:41,129 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:41,129 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:41,181 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f2745987e524405888f3b81834a980fb 2024-11-20T05:16:41,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/629bdcc3b44745548f15615cf4f522f0 is 50, key is test_row_0/C:col10/1732079799515/Put/seqid=0 2024-11-20T05:16:41,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742284_1460 (size=12151) 2024-11-20T05:16:41,195 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/629bdcc3b44745548f15615cf4f522f0 2024-11-20T05:16:41,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/13c1a61154bc4ca5ad810c46fd292013 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13c1a61154bc4ca5ad810c46fd292013 2024-11-20T05:16:41,206 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13c1a61154bc4ca5ad810c46fd292013, entries=150, sequenceid=223, filesize=11.9 K 2024-11-20T05:16:41,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/f2745987e524405888f3b81834a980fb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f2745987e524405888f3b81834a980fb 2024-11-20T05:16:41,210 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f2745987e524405888f3b81834a980fb, entries=150, sequenceid=223, filesize=11.9 K 2024-11-20T05:16:41,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/629bdcc3b44745548f15615cf4f522f0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/629bdcc3b44745548f15615cf4f522f0 2024-11-20T05:16:41,215 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/629bdcc3b44745548f15615cf4f522f0, entries=150, sequenceid=223, filesize=11.9 K 2024-11-20T05:16:41,216 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for bef9917e0871717b985dc9ca0a53e5ad in 853ms, sequenceid=223, compaction requested=false 2024-11-20T05:16:41,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:41,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:41,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-20T05:16:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-20T05:16:41,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-20T05:16:41,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2970 sec 2024-11-20T05:16:41,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.3030 sec 2024-11-20T05:16:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:41,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:16:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:41,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ee2e57c9cf3943369f0cbd0f8d6c333a is 50, key is test_row_0/A:col10/1732079801321/Put/seqid=0 2024-11-20T05:16:41,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742285_1461 (size=16931) 2024-11-20T05:16:41,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ee2e57c9cf3943369f0cbd0f8d6c333a 2024-11-20T05:16:41,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/2ae82078da6c4fad917e6af5234ff899 is 50, key is test_row_0/B:col10/1732079801321/Put/seqid=0 2024-11-20T05:16:41,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742286_1462 (size=12151) 2024-11-20T05:16:41,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/2ae82078da6c4fad917e6af5234ff899 2024-11-20T05:16:41,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/a790b93c74e049e8a03f61930f8e7ad6 is 50, key is test_row_0/C:col10/1732079801321/Put/seqid=0 2024-11-20T05:16:41,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742287_1463 (size=12151) 2024-11-20T05:16:41,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079861350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079861351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079861354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079861357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079861467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079861468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079861468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079861477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079861687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079861688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079861688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079861688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079861727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:41,736 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:41,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/a790b93c74e049e8a03f61930f8e7ad6 2024-11-20T05:16:41,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ee2e57c9cf3943369f0cbd0f8d6c333a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ee2e57c9cf3943369f0cbd0f8d6c333a 2024-11-20T05:16:41,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ee2e57c9cf3943369f0cbd0f8d6c333a, entries=250, sequenceid=242, filesize=16.5 K 2024-11-20T05:16:41,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/2ae82078da6c4fad917e6af5234ff899 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2ae82078da6c4fad917e6af5234ff899 2024-11-20T05:16:41,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2ae82078da6c4fad917e6af5234ff899, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T05:16:41,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/a790b93c74e049e8a03f61930f8e7ad6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a790b93c74e049e8a03f61930f8e7ad6 2024-11-20T05:16:41,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a790b93c74e049e8a03f61930f8e7ad6, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T05:16:41,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for bef9917e0871717b985dc9ca0a53e5ad in 457ms, sequenceid=242, compaction requested=true 2024-11-20T05:16:41,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:41,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:41,780 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:41,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:41,780 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:41,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:41,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:41,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:41,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:41,781 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:41,781 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:41,781 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:41,781 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:41,781 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:41,781 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:41,781 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f1e8833706224bf7b6d69e1dd1e2ceb1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f2745987e524405888f3b81834a980fb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2ae82078da6c4fad917e6af5234ff899] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=36.1 K 2024-11-20T05:16:41,781 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/fb39df31e41a48bb948bacc68be29e93, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13c1a61154bc4ca5ad810c46fd292013, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ee2e57c9cf3943369f0cbd0f8d6c333a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=40.8 K 2024-11-20T05:16:41,781 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb39df31e41a48bb948bacc68be29e93, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732079798462 2024-11-20T05:16:41,782 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e8833706224bf7b6d69e1dd1e2ceb1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732079798462 2024-11-20T05:16:41,782 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13c1a61154bc4ca5ad810c46fd292013, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732079799515 2024-11-20T05:16:41,782 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f2745987e524405888f3b81834a980fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732079799515 2024-11-20T05:16:41,782 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee2e57c9cf3943369f0cbd0f8d6c333a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732079800654 2024-11-20T05:16:41,782 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae82078da6c4fad917e6af5234ff899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732079800670 2024-11-20T05:16:41,790 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:41,790 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5a0aefdf7ccd4e2ba1518b24a57df4a3 is 50, key is test_row_0/B:col10/1732079801321/Put/seqid=0 2024-11-20T05:16:41,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742288_1464 (size=12765) 2024-11-20T05:16:41,803 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#393 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:41,803 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/c93bfa7e9ebe46d1b5a0050b142e06c9 is 50, key is test_row_0/A:col10/1732079801321/Put/seqid=0 2024-11-20T05:16:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742289_1465 (size=12765) 2024-11-20T05:16:41,815 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5a0aefdf7ccd4e2ba1518b24a57df4a3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5a0aefdf7ccd4e2ba1518b24a57df4a3 2024-11-20T05:16:41,816 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/c93bfa7e9ebe46d1b5a0050b142e06c9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/c93bfa7e9ebe46d1b5a0050b142e06c9 2024-11-20T05:16:41,818 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into 5a0aefdf7ccd4e2ba1518b24a57df4a3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:41,818 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:41,818 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=13, startTime=1732079801780; duration=0sec 2024-11-20T05:16:41,818 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:41,818 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:41,818 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:16:41,818 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:16:41,819 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:41,819 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:41,819 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/c90d18397ac1441baa77e94e6d32cc32, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/629bdcc3b44745548f15615cf4f522f0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a790b93c74e049e8a03f61930f8e7ad6] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=36.1 K 2024-11-20T05:16:41,819 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c90d18397ac1441baa77e94e6d32cc32, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732079798462 2024-11-20T05:16:41,819 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into c93bfa7e9ebe46d1b5a0050b142e06c9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:41,819 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:41,819 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=13, startTime=1732079801780; duration=0sec 2024-11-20T05:16:41,819 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:41,819 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 629bdcc3b44745548f15615cf4f522f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732079799515 2024-11-20T05:16:41,820 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:41,820 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a790b93c74e049e8a03f61930f8e7ad6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732079800670 2024-11-20T05:16:41,827 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:41,827 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/9cb67a9f7e0348bebaac5b7faaefc22b is 50, key is test_row_0/C:col10/1732079801321/Put/seqid=0 2024-11-20T05:16:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742290_1466 (size=12765) 2024-11-20T05:16:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:42,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:16:42,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:42,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:42,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:42,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:42,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:42,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:42,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/252a2e8f05b048c5a9c001a3512cc237 is 50, key is test_row_0/A:col10/1732079801349/Put/seqid=0 2024-11-20T05:16:42,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742291_1467 (size=14641) 2024-11-20T05:16:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079862027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079862027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079862033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079862035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T05:16:42,088 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:42,088 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-20T05:16:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T05:16:42,090 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:42,090 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:42,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:42,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079862137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079862137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079862147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079862147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T05:16:42,238 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/9cb67a9f7e0348bebaac5b7faaefc22b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9cb67a9f7e0348bebaac5b7faaefc22b 2024-11-20T05:16:42,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:42,242 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into 9cb67a9f7e0348bebaac5b7faaefc22b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,242 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:42,242 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=13, startTime=1732079801780; duration=0sec 2024-11-20T05:16:42,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,242 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:42,242 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079862357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079862357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079862357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079862367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:42,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:42,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T05:16:42,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/252a2e8f05b048c5a9c001a3512cc237 2024-11-20T05:16:42,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9952d7ebe46b4ba8a3058d0489664528 is 50, key is test_row_0/B:col10/1732079801349/Put/seqid=0 2024-11-20T05:16:42,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742292_1468 (size=12251) 2024-11-20T05:16:42,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079862667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079862667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079862667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079862677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:42,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T05:16:42,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9952d7ebe46b4ba8a3058d0489664528 2024-11-20T05:16:42,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/4e9fb4d0667d4c7b835131dab89b2e42 is 50, key is test_row_0/C:col10/1732079801349/Put/seqid=0 2024-11-20T05:16:42,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742293_1469 (size=12251) 2024-11-20T05:16:42,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:42,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:43,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:43,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:43,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:43,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:43,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:43,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:43,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:43,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079863187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079863187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079863187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079863197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T05:16:43,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/4e9fb4d0667d4c7b835131dab89b2e42 2024-11-20T05:16:43,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/252a2e8f05b048c5a9c001a3512cc237 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/252a2e8f05b048c5a9c001a3512cc237 2024-11-20T05:16:43,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/252a2e8f05b048c5a9c001a3512cc237, entries=200, sequenceid=261, filesize=14.3 K 2024-11-20T05:16:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9952d7ebe46b4ba8a3058d0489664528 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9952d7ebe46b4ba8a3058d0489664528 2024-11-20T05:16:43,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9952d7ebe46b4ba8a3058d0489664528, entries=150, sequenceid=261, filesize=12.0 K 2024-11-20T05:16:43,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/4e9fb4d0667d4c7b835131dab89b2e42 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4e9fb4d0667d4c7b835131dab89b2e42 2024-11-20T05:16:43,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4e9fb4d0667d4c7b835131dab89b2e42, entries=150, sequenceid=261, filesize=12.0 K 2024-11-20T05:16:43,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for bef9917e0871717b985dc9ca0a53e5ad in 1236ms, sequenceid=261, compaction requested=false 2024-11-20T05:16:43,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:43,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T05:16:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:43,308 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T05:16:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:43,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/7c5c7e7b5912433c8afaa99c4021d335 is 50, key is test_row_0/A:col10/1732079802024/Put/seqid=0 2024-11-20T05:16:43,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742294_1470 (size=12301) 2024-11-20T05:16:43,716 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/7c5c7e7b5912433c8afaa99c4021d335 2024-11-20T05:16:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bff2ed51615f44e4a0028f11694d6029 is 50, key is test_row_0/B:col10/1732079802024/Put/seqid=0 2024-11-20T05:16:43,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742295_1471 (size=12301) 2024-11-20T05:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:43,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:43,790 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T05:16:43,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079863858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:43,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079863967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:43,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:44,137 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bff2ed51615f44e4a0028f11694d6029 2024-11-20T05:16:44,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/2cb3da3625e94a53b25c5a2193625e06 is 50, key is test_row_0/C:col10/1732079802024/Put/seqid=0 2024-11-20T05:16:44,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742296_1472 (size=12301) 2024-11-20T05:16:44,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079864187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:44,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079864207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:44,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079864207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:44,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079864208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:44,214 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,214 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:44,214 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079864217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:44,226 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T05:16:44,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079864507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:44,546 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/2cb3da3625e94a53b25c5a2193625e06 2024-11-20T05:16:44,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/7c5c7e7b5912433c8afaa99c4021d335 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/7c5c7e7b5912433c8afaa99c4021d335 2024-11-20T05:16:44,552 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/7c5c7e7b5912433c8afaa99c4021d335, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T05:16:44,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bff2ed51615f44e4a0028f11694d6029 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bff2ed51615f44e4a0028f11694d6029 2024-11-20T05:16:44,556 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bff2ed51615f44e4a0028f11694d6029, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T05:16:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/2cb3da3625e94a53b25c5a2193625e06 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/2cb3da3625e94a53b25c5a2193625e06 2024-11-20T05:16:44,559 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/2cb3da3625e94a53b25c5a2193625e06, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T05:16:44,560 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for bef9917e0871717b985dc9ca0a53e5ad in 1252ms, sequenceid=277, compaction requested=true 2024-11-20T05:16:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-20T05:16:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-20T05:16:44,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T05:16:44,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4710 sec 2024-11-20T05:16:44,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.4750 sec 2024-11-20T05:16:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:45,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T05:16:45,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:45,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:45,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:45,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:45,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:45,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:45,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/15eb049300a0449fbc30279059c574d3 is 50, key is test_row_0/A:col10/1732079803857/Put/seqid=0 2024-11-20T05:16:45,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742297_1473 (size=14741) 2024-11-20T05:16:45,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/15eb049300a0449fbc30279059c574d3 2024-11-20T05:16:45,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/884e17988a4c4991a795a04fe7721279 is 50, key is test_row_0/B:col10/1732079803857/Put/seqid=0 2024-11-20T05:16:45,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742298_1474 (size=12301) 2024-11-20T05:16:45,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079865137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:45,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079865247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:45,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/884e17988a4c4991a795a04fe7721279 2024-11-20T05:16:45,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/3e29afde246548efb72315dfd91dd31a is 50, key is test_row_0/C:col10/1732079803857/Put/seqid=0 2024-11-20T05:16:45,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742299_1475 (size=12301) 2024-11-20T05:16:45,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079865467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:45,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079865787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:45,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/3e29afde246548efb72315dfd91dd31a 2024-11-20T05:16:45,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/15eb049300a0449fbc30279059c574d3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/15eb049300a0449fbc30279059c574d3 2024-11-20T05:16:45,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/15eb049300a0449fbc30279059c574d3, entries=200, sequenceid=295, filesize=14.4 K 2024-11-20T05:16:45,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/884e17988a4c4991a795a04fe7721279 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/884e17988a4c4991a795a04fe7721279 2024-11-20T05:16:45,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/884e17988a4c4991a795a04fe7721279, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T05:16:45,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/3e29afde246548efb72315dfd91dd31a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/3e29afde246548efb72315dfd91dd31a 2024-11-20T05:16:45,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/3e29afde246548efb72315dfd91dd31a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T05:16:45,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for bef9917e0871717b985dc9ca0a53e5ad in 844ms, sequenceid=295, compaction requested=true 2024-11-20T05:16:45,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:45,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:45,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:45,877 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:16:45,877 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:16:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:45,878 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54448 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49618 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:16:45,879 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:45,879 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:45,879 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/c93bfa7e9ebe46d1b5a0050b142e06c9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/252a2e8f05b048c5a9c001a3512cc237, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/7c5c7e7b5912433c8afaa99c4021d335, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/15eb049300a0449fbc30279059c574d3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=53.2 K 2024-11-20T05:16:45,879 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5a0aefdf7ccd4e2ba1518b24a57df4a3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9952d7ebe46b4ba8a3058d0489664528, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bff2ed51615f44e4a0028f11694d6029, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/884e17988a4c4991a795a04fe7721279] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=48.5 K 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a0aefdf7ccd4e2ba1518b24a57df4a3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732079800670 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c93bfa7e9ebe46d1b5a0050b142e06c9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732079800670 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9952d7ebe46b4ba8a3058d0489664528, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732079801349 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 252a2e8f05b048c5a9c001a3512cc237, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732079801349 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bff2ed51615f44e4a0028f11694d6029, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079802024 2024-11-20T05:16:45,879 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5c7e7b5912433c8afaa99c4021d335, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079802024 2024-11-20T05:16:45,880 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15eb049300a0449fbc30279059c574d3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732079803844 2024-11-20T05:16:45,880 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 884e17988a4c4991a795a04fe7721279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732079803846 2024-11-20T05:16:45,886 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:45,886 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/3c4d2e0637d643afa82cde51ba05d178 is 50, key is test_row_0/A:col10/1732079803857/Put/seqid=0 2024-11-20T05:16:45,886 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:45,887 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bb7420cc4559494ca1602556412aa394 is 50, key is test_row_0/B:col10/1732079803857/Put/seqid=0 2024-11-20T05:16:45,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742300_1476 (size=13051) 2024-11-20T05:16:45,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742301_1477 (size=13051) 2024-11-20T05:16:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:46,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T05:16:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:46,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/454b596352cc49fb94ffdf48dfb93035 is 50, key is test_row_0/A:col10/1732079806232/Put/seqid=0 2024-11-20T05:16:46,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742302_1478 (size=14741) 2024-11-20T05:16:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T05:16:46,247 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:46,247 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-20T05:16:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T05:16:46,249 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:46,249 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:46,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079866263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079866263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079866264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079866269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,299 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/3c4d2e0637d643afa82cde51ba05d178 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/3c4d2e0637d643afa82cde51ba05d178 2024-11-20T05:16:46,299 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bb7420cc4559494ca1602556412aa394 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bb7420cc4559494ca1602556412aa394 2024-11-20T05:16:46,302 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into bb7420cc4559494ca1602556412aa394(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:46,302 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into 3c4d2e0637d643afa82cde51ba05d178(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:46,303 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=12, startTime=1732079805877; duration=0sec 2024-11-20T05:16:46,303 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=12, startTime=1732079805877; duration=0sec 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:16:46,303 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49618 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:16:46,304 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:46,304 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,304 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9cb67a9f7e0348bebaac5b7faaefc22b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4e9fb4d0667d4c7b835131dab89b2e42, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/2cb3da3625e94a53b25c5a2193625e06, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/3e29afde246548efb72315dfd91dd31a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=48.5 K 2024-11-20T05:16:46,304 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb67a9f7e0348bebaac5b7faaefc22b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732079800670 2024-11-20T05:16:46,304 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9fb4d0667d4c7b835131dab89b2e42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732079801349 2024-11-20T05:16:46,304 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb3da3625e94a53b25c5a2193625e06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732079802024 2024-11-20T05:16:46,305 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e29afde246548efb72315dfd91dd31a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732079803846 2024-11-20T05:16:46,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079866297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,312 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:46,313 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/4cc6f090984f405797aec81f590ec3a1 is 50, key is test_row_0/C:col10/1732079803857/Put/seqid=0 2024-11-20T05:16:46,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742303_1479 (size=13051) 2024-11-20T05:16:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T05:16:46,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079866377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079866387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079866388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079866388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T05:16:46,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079866587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079866597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079866597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079866597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/454b596352cc49fb94ffdf48dfb93035 2024-11-20T05:16:46,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/0fc5224864bc499e9e0ac596e4d62d19 is 50, key is test_row_0/B:col10/1732079806232/Put/seqid=0 2024-11-20T05:16:46,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742304_1480 (size=12301) 2024-11-20T05:16:46,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,721 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/4cc6f090984f405797aec81f590ec3a1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4cc6f090984f405797aec81f590ec3a1 2024-11-20T05:16:46,725 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into 4cc6f090984f405797aec81f590ec3a1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:46,725 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:46,725 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=12, startTime=1732079805878; duration=0sec 2024-11-20T05:16:46,725 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:46,725 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:46,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:46,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T05:16:46,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079866897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079866917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079866917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079866917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:47,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:47,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:47,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/0fc5224864bc499e9e0ac596e4d62d19 2024-11-20T05:16:47,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/577bd6141c9c4bd497822b76765f0903 is 50, key is test_row_0/C:col10/1732079806232/Put/seqid=0 2024-11-20T05:16:47,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742305_1481 (size=12301) 2024-11-20T05:16:47,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079867317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:47,323 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T05:16:47,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079867407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079867427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079867437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079867438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:47,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/577bd6141c9c4bd497822b76765f0903 2024-11-20T05:16:47,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/454b596352cc49fb94ffdf48dfb93035 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/454b596352cc49fb94ffdf48dfb93035 2024-11-20T05:16:47,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/454b596352cc49fb94ffdf48dfb93035, entries=200, sequenceid=310, filesize=14.4 K 2024-11-20T05:16:47,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/0fc5224864bc499e9e0ac596e4d62d19 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/0fc5224864bc499e9e0ac596e4d62d19 2024-11-20T05:16:47,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/0fc5224864bc499e9e0ac596e4d62d19, entries=150, sequenceid=310, filesize=12.0 K 2024-11-20T05:16:47,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/577bd6141c9c4bd497822b76765f0903 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/577bd6141c9c4bd497822b76765f0903 2024-11-20T05:16:47,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:47,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/577bd6141c9c4bd497822b76765f0903, entries=150, sequenceid=310, filesize=12.0 K 2024-11-20T05:16:47,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for bef9917e0871717b985dc9ca0a53e5ad in 1241ms, sequenceid=310, compaction requested=false 2024-11-20T05:16:47,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:47,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T05:16:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:47,623 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:16:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/9fc8e982754849ef988333a49c1b6477 is 50, key is test_row_0/A:col10/1732079806267/Put/seqid=0 2024-11-20T05:16:47,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742306_1482 (size=12301) 2024-11-20T05:16:48,031 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/9fc8e982754849ef988333a49c1b6477 2024-11-20T05:16:48,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/c0c6b9af8ff34cb18a2deead2b36aff6 is 50, key is test_row_0/B:col10/1732079806267/Put/seqid=0 2024-11-20T05:16:48,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742307_1483 (size=12301) 2024-11-20T05:16:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T05:16:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:48,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. as already flushing 2024-11-20T05:16:48,448 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/c0c6b9af8ff34cb18a2deead2b36aff6 2024-11-20T05:16:48,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/e2bde45c077d457a8e42ea00557d5760 is 50, key is test_row_0/C:col10/1732079806267/Put/seqid=0 2024-11-20T05:16:48,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742308_1484 (size=12301) 2024-11-20T05:16:48,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079868467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079868469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079868470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079868472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079868577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079868587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079868587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079868587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079868787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079868807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079868807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079868807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:48,858 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/e2bde45c077d457a8e42ea00557d5760 2024-11-20T05:16:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/9fc8e982754849ef988333a49c1b6477 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9fc8e982754849ef988333a49c1b6477 2024-11-20T05:16:48,863 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9fc8e982754849ef988333a49c1b6477, entries=150, sequenceid=330, filesize=12.0 K 2024-11-20T05:16:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/c0c6b9af8ff34cb18a2deead2b36aff6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/c0c6b9af8ff34cb18a2deead2b36aff6 2024-11-20T05:16:48,866 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/c0c6b9af8ff34cb18a2deead2b36aff6, entries=150, sequenceid=330, filesize=12.0 K 2024-11-20T05:16:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/e2bde45c077d457a8e42ea00557d5760 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e2bde45c077d457a8e42ea00557d5760 2024-11-20T05:16:48,870 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e2bde45c077d457a8e42ea00557d5760, entries=150, sequenceid=330, filesize=12.0 K 2024-11-20T05:16:48,871 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for bef9917e0871717b985dc9ca0a53e5ad in 1247ms, sequenceid=330, compaction requested=true 2024-11-20T05:16:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-20T05:16:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-20T05:16:48,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T05:16:48,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6230 sec 2024-11-20T05:16:48,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6260 sec 2024-11-20T05:16:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:49,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:16:49,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:49,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:49,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:49,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:49,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:49,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:49,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ba1194fde28446c99e4601805619ea88 is 50, key is test_row_0/A:col10/1732079808468/Put/seqid=0 2024-11-20T05:16:49,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742309_1485 (size=14741) 2024-11-20T05:16:49,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079869152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079869153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079869155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079869156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079869267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079869277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079869277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079869277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41852 deadline: 1732079869327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,334 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in bef9917e0871717b985dc9ca0a53e5ad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:16:49,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079869487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079869488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079869488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079869488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ba1194fde28446c99e4601805619ea88 2024-11-20T05:16:49,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/4736206f1593479181082452ce73cfd0 is 50, key is test_row_0/B:col10/1732079808468/Put/seqid=0 2024-11-20T05:16:49,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742310_1486 (size=12301) 2024-11-20T05:16:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079869807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079869807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079869808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079869808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:49,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/4736206f1593479181082452ce73cfd0 2024-11-20T05:16:49,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/b9a1806a871b42d2a2a31bb085339aaa is 50, key is test_row_0/C:col10/1732079808468/Put/seqid=0 2024-11-20T05:16:49,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742311_1487 (size=12301) 2024-11-20T05:16:50,327 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-20T05:16:50,327 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:50,327 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,327 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:50,333 INFO [Thread-1876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1876. 2024-11-20T05:16:50,333 DEBUG [Thread-1876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:50,333 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,333 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:50,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:50,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41830 deadline: 1732079870327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:50,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:50,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41898 deadline: 1732079870327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:50,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41868 deadline: 1732079870327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:50,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/b9a1806a871b42d2a2a31bb085339aaa 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:50,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:50,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41862 deadline: 1732079870327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 , the old value is region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., hostname=51d5d2011a1c,44753,1732079655401, seqNum=2 because the exception is null or not the one we care about 2024-11-20T05:16:50,337 INFO [Thread-1874 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1874. 2024-11-20T05:16:50,337 DEBUG [Thread-1874 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:50,337 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,337 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:50,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ba1194fde28446c99e4601805619ea88 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ba1194fde28446c99e4601805619ea88 2024-11-20T05:16:50,340 INFO [Thread-1868 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1868. 2024-11-20T05:16:50,340 DEBUG [Thread-1868 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:50,340 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,341 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:50,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ba1194fde28446c99e4601805619ea88, entries=200, sequenceid=346, filesize=14.4 K 2024-11-20T05:16:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/4736206f1593479181082452ce73cfd0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4736206f1593479181082452ce73cfd0 2024-11-20T05:16:50,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4736206f1593479181082452ce73cfd0, entries=150, sequenceid=346, filesize=12.0 K 2024-11-20T05:16:50,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/b9a1806a871b42d2a2a31bb085339aaa as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b9a1806a871b42d2a2a31bb085339aaa 2024-11-20T05:16:50,347 INFO [Thread-1872 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1872. 2024-11-20T05:16:50,347 DEBUG [Thread-1872 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:50,347 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,347 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:50,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:50,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b9a1806a871b42d2a2a31bb085339aaa, entries=150, sequenceid=346, filesize=12.0 K 2024-11-20T05:16:50,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for bef9917e0871717b985dc9ca0a53e5ad in 1245ms, sequenceid=346, compaction requested=true 2024-11-20T05:16:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:16:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:16:50,350 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:16:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bef9917e0871717b985dc9ca0a53e5ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:16:50,350 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:16:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/B is initiating minor compaction (all files) 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/A is initiating minor compaction (all files) 2024-11-20T05:16:50,351 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/B in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:50,351 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/A in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:50,351 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bb7420cc4559494ca1602556412aa394, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/0fc5224864bc499e9e0ac596e4d62d19, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/c0c6b9af8ff34cb18a2deead2b36aff6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4736206f1593479181082452ce73cfd0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=48.8 K 2024-11-20T05:16:50,351 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/3c4d2e0637d643afa82cde51ba05d178, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/454b596352cc49fb94ffdf48dfb93035, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9fc8e982754849ef988333a49c1b6477, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ba1194fde28446c99e4601805619ea88] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=53.5 K 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bb7420cc4559494ca1602556412aa394, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732079803846 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4d2e0637d643afa82cde51ba05d178, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732079803846 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc5224864bc499e9e0ac596e4d62d19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732079805067 2024-11-20T05:16:50,351 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454b596352cc49fb94ffdf48dfb93035, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732079805067 2024-11-20T05:16:50,352 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c6b9af8ff34cb18a2deead2b36aff6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732079806245 2024-11-20T05:16:50,352 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc8e982754849ef988333a49c1b6477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732079806245 2024-11-20T05:16:50,352 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4736206f1593479181082452ce73cfd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732079808466 2024-11-20T05:16:50,352 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1194fde28446c99e4601805619ea88, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732079808466 2024-11-20T05:16:50,358 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:50,358 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9a0be5b18bfe42de977962bf7a24776c is 50, key is test_row_0/B:col10/1732079808468/Put/seqid=0 2024-11-20T05:16:50,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742312_1488 (size=13187) 2024-11-20T05:16:50,362 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:50,362 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ade4e29f674b449e9182b84f8ee533c6 is 50, key is test_row_0/A:col10/1732079808468/Put/seqid=0 2024-11-20T05:16:50,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742313_1489 (size=13187) 2024-11-20T05:16:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T05:16:50,407 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:50,766 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/9a0be5b18bfe42de977962bf7a24776c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9a0be5b18bfe42de977962bf7a24776c 2024-11-20T05:16:50,769 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/ade4e29f674b449e9182b84f8ee533c6 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ade4e29f674b449e9182b84f8ee533c6 2024-11-20T05:16:50,769 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/B of bef9917e0871717b985dc9ca0a53e5ad into 9a0be5b18bfe42de977962bf7a24776c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:50,769 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:50,769 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/B, priority=12, startTime=1732079810350; duration=0sec 2024-11-20T05:16:50,769 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:16:50,769 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:B 2024-11-20T05:16:50,769 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:16:50,770 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:16:50,770 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): bef9917e0871717b985dc9ca0a53e5ad/C is initiating minor compaction (all files) 2024-11-20T05:16:50,770 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bef9917e0871717b985dc9ca0a53e5ad/C in TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:50,770 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4cc6f090984f405797aec81f590ec3a1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/577bd6141c9c4bd497822b76765f0903, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e2bde45c077d457a8e42ea00557d5760, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b9a1806a871b42d2a2a31bb085339aaa] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp, totalSize=48.8 K 2024-11-20T05:16:50,771 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc6f090984f405797aec81f590ec3a1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732079803846 2024-11-20T05:16:50,771 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 577bd6141c9c4bd497822b76765f0903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732079805067 2024-11-20T05:16:50,771 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e2bde45c077d457a8e42ea00557d5760, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732079806245 2024-11-20T05:16:50,771 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a1806a871b42d2a2a31bb085339aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732079808466 2024-11-20T05:16:50,773 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/A of bef9917e0871717b985dc9ca0a53e5ad into ade4e29f674b449e9182b84f8ee533c6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:50,773 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:50,773 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/A, priority=12, startTime=1732079810350; duration=0sec 2024-11-20T05:16:50,773 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:50,773 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:A 2024-11-20T05:16:50,778 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bef9917e0871717b985dc9ca0a53e5ad#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:16:50,779 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/b97475fddafe459b8eff9936c46d93fe is 50, key is test_row_0/C:col10/1732079808468/Put/seqid=0 2024-11-20T05:16:50,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742314_1490 (size=13187) 2024-11-20T05:16:51,193 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/b97475fddafe459b8eff9936c46d93fe as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b97475fddafe459b8eff9936c46d93fe 2024-11-20T05:16:51,200 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bef9917e0871717b985dc9ca0a53e5ad/C of bef9917e0871717b985dc9ca0a53e5ad into b97475fddafe459b8eff9936c46d93fe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:16:51,200 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:51,200 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., storeName=bef9917e0871717b985dc9ca0a53e5ad/C, priority=12, startTime=1732079810350; duration=0sec 2024-11-20T05:16:51,200 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:16:51,200 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bef9917e0871717b985dc9ca0a53e5ad:C 2024-11-20T05:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:51,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T05:16:51,354 INFO [Thread-1857 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1857. 2024-11-20T05:16:51,354 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-20T05:16:51,354 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:51,354 DEBUG [Thread-1857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:51,354 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,354 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,354 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,354 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,354 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-20T05:16:51,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:51,355 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:51,355 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:51,355 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:51,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:51,357 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-11-20T05:16:51,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:51,357 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:51,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:51,357 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:51,357 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:51,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:51,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:51,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/be9901b7061f442c8b87b3ef125516e7 is 50, key is test_row_0/A:col10/1732079811354/Put/seqid=0 2024-11-20T05:16:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742315_1491 (size=12301) 2024-11-20T05:16:51,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/be9901b7061f442c8b87b3ef125516e7 2024-11-20T05:16:51,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5b539b64169445649f84932c09d4333b is 50, key is test_row_0/B:col10/1732079811354/Put/seqid=0 2024-11-20T05:16:51,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742316_1492 (size=12301) 2024-11-20T05:16:52,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5b539b64169445649f84932c09d4333b 2024-11-20T05:16:52,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/44c6f330c37e4eaabed15bb0fb562f5a is 50, key is test_row_0/C:col10/1732079811354/Put/seqid=0 2024-11-20T05:16:52,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742317_1493 (size=12301) 2024-11-20T05:16:52,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/44c6f330c37e4eaabed15bb0fb562f5a 2024-11-20T05:16:52,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/be9901b7061f442c8b87b3ef125516e7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/be9901b7061f442c8b87b3ef125516e7 2024-11-20T05:16:52,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/be9901b7061f442c8b87b3ef125516e7, entries=150, sequenceid=369, filesize=12.0 K 2024-11-20T05:16:52,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/5b539b64169445649f84932c09d4333b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5b539b64169445649f84932c09d4333b 2024-11-20T05:16:52,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5b539b64169445649f84932c09d4333b, entries=150, sequenceid=369, filesize=12.0 K 2024-11-20T05:16:52,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/44c6f330c37e4eaabed15bb0fb562f5a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/44c6f330c37e4eaabed15bb0fb562f5a 2024-11-20T05:16:52,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/44c6f330c37e4eaabed15bb0fb562f5a, entries=150, sequenceid=369, filesize=12.0 K 2024-11-20T05:16:52,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for bef9917e0871717b985dc9ca0a53e5ad in 1273ms, sequenceid=369, compaction requested=false 2024-11-20T05:16:52,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bef9917e0871717b985dc9ca0a53e5ad: 2024-11-20T05:16:53,372 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-11-20T05:16:53,372 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:16:53,372 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:53,372 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:53,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1120 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3359 rows 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3383 rows 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3384 rows 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3372 rows 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1131 2024-11-20T05:16:53,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3392 rows 2024-11-20T05:16:53,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:16:53,374 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:16:53,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:53,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:53,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:16:53,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T05:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T05:16:53,383 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079813382"}]},"ts":"1732079813382"} 2024-11-20T05:16:53,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T05:16:53,385 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T05:16:53,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:16:53,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, UNASSIGN}] 2024-11-20T05:16:53,389 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, UNASSIGN 2024-11-20T05:16:53,390 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=bef9917e0871717b985dc9ca0a53e5ad, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:53,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, UNASSIGN because future has completed 2024-11-20T05:16:53,392 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:16:53,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:16:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T05:16:53,545 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:53,545 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:16:53,545 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing bef9917e0871717b985dc9ca0a53e5ad, disabling compactions & flushes 2024-11-20T05:16:53,545 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:53,545 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:53,545 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. after waiting 0 ms 2024-11-20T05:16:53,545 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:53,546 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing bef9917e0871717b985dc9ca0a53e5ad 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-20T05:16:53,546 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=A 2024-11-20T05:16:53,546 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:53,546 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=B 2024-11-20T05:16:53,546 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:53,546 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bef9917e0871717b985dc9ca0a53e5ad, store=C 2024-11-20T05:16:53,546 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:53,549 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/06534eb2d4cb45c38396a9f86820c758 is 50, key is test_row_1/A:col10/1732079813368/Put/seqid=0 2024-11-20T05:16:53,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742318_1494 (size=7415) 2024-11-20T05:16:53,559 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/06534eb2d4cb45c38396a9f86820c758 2024-11-20T05:16:53,566 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bfbeabf0b04a428caed22a2d00959a62 is 50, key is test_row_1/B:col10/1732079813368/Put/seqid=0 2024-11-20T05:16:53,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742319_1495 (size=7415) 2024-11-20T05:16:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T05:16:53,971 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bfbeabf0b04a428caed22a2d00959a62 2024-11-20T05:16:53,982 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/e4ce2d81b6d344ef8753312e73e48671 is 50, key is test_row_1/C:col10/1732079813368/Put/seqid=0 2024-11-20T05:16:53,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742320_1496 (size=7415) 2024-11-20T05:16:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T05:16:54,387 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/e4ce2d81b6d344ef8753312e73e48671 2024-11-20T05:16:54,440 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/A/06534eb2d4cb45c38396a9f86820c758 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/06534eb2d4cb45c38396a9f86820c758 2024-11-20T05:16:54,446 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/06534eb2d4cb45c38396a9f86820c758, entries=50, sequenceid=373, filesize=7.2 K 2024-11-20T05:16:54,447 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/B/bfbeabf0b04a428caed22a2d00959a62 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bfbeabf0b04a428caed22a2d00959a62 2024-11-20T05:16:54,451 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bfbeabf0b04a428caed22a2d00959a62, entries=50, sequenceid=373, filesize=7.2 K 2024-11-20T05:16:54,452 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/.tmp/C/e4ce2d81b6d344ef8753312e73e48671 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e4ce2d81b6d344ef8753312e73e48671 2024-11-20T05:16:54,455 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e4ce2d81b6d344ef8753312e73e48671, entries=50, sequenceid=373, filesize=7.2 K 2024-11-20T05:16:54,456 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for bef9917e0871717b985dc9ca0a53e5ad in 911ms, sequenceid=373, compaction requested=true 2024-11-20T05:16:54,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/441ed964d9b44d0fa43b309ebfec30f9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/1fa1a90d8711443892eb4d772568b661, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0870ef5ebf9846c08d5db6f92c1cf0ad, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/85aa61c2b1ac4615b047b3ace5927006, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0a556276deba4ea39912a5182dc8fc14, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/354e707e7411457d83a0fed8ed6b9f33, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/de0ae086cefb42bb85b7ca7a9b634857, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ff1e743dbd134c73937eec434278d635, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/e0d097d645944e76a11204ab36b49d51, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/95da6be115b04e5a9fd991e305752abb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/75cc31452eef42a297cb32f87cfc72b7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/637052a1ae0b4a18a96d4567f29388f4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13482fecb1ce41edbc4fa5cc7239b18c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/f63c5a5f821a4d64935faae70daab0ff, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/fb39df31e41a48bb948bacc68be29e93, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13c1a61154bc4ca5ad810c46fd292013, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ee2e57c9cf3943369f0cbd0f8d6c333a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/c93bfa7e9ebe46d1b5a0050b142e06c9, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/252a2e8f05b048c5a9c001a3512cc237, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/7c5c7e7b5912433c8afaa99c4021d335, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/15eb049300a0449fbc30279059c574d3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/3c4d2e0637d643afa82cde51ba05d178, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/454b596352cc49fb94ffdf48dfb93035, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9fc8e982754849ef988333a49c1b6477, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ba1194fde28446c99e4601805619ea88] to archive 2024-11-20T05:16:54,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:54,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/441ed964d9b44d0fa43b309ebfec30f9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/441ed964d9b44d0fa43b309ebfec30f9 2024-11-20T05:16:54,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/1fa1a90d8711443892eb4d772568b661 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/1fa1a90d8711443892eb4d772568b661 2024-11-20T05:16:54,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0870ef5ebf9846c08d5db6f92c1cf0ad to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0870ef5ebf9846c08d5db6f92c1cf0ad 2024-11-20T05:16:54,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/85aa61c2b1ac4615b047b3ace5927006 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/85aa61c2b1ac4615b047b3ace5927006 2024-11-20T05:16:54,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0a556276deba4ea39912a5182dc8fc14 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/0a556276deba4ea39912a5182dc8fc14 2024-11-20T05:16:54,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/354e707e7411457d83a0fed8ed6b9f33 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/354e707e7411457d83a0fed8ed6b9f33 2024-11-20T05:16:54,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/de0ae086cefb42bb85b7ca7a9b634857 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/de0ae086cefb42bb85b7ca7a9b634857 2024-11-20T05:16:54,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ff1e743dbd134c73937eec434278d635 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ff1e743dbd134c73937eec434278d635 2024-11-20T05:16:54,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/e0d097d645944e76a11204ab36b49d51 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/e0d097d645944e76a11204ab36b49d51 2024-11-20T05:16:54,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/95da6be115b04e5a9fd991e305752abb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/95da6be115b04e5a9fd991e305752abb 2024-11-20T05:16:54,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/75cc31452eef42a297cb32f87cfc72b7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/75cc31452eef42a297cb32f87cfc72b7 2024-11-20T05:16:54,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/637052a1ae0b4a18a96d4567f29388f4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/637052a1ae0b4a18a96d4567f29388f4 2024-11-20T05:16:54,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9d7b7a8e80f24ccc8215ff2a19fc9bfa 2024-11-20T05:16:54,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13482fecb1ce41edbc4fa5cc7239b18c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13482fecb1ce41edbc4fa5cc7239b18c 2024-11-20T05:16:54,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/f63c5a5f821a4d64935faae70daab0ff to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/f63c5a5f821a4d64935faae70daab0ff 2024-11-20T05:16:54,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/fb39df31e41a48bb948bacc68be29e93 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/fb39df31e41a48bb948bacc68be29e93 2024-11-20T05:16:54,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13c1a61154bc4ca5ad810c46fd292013 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/13c1a61154bc4ca5ad810c46fd292013 2024-11-20T05:16:54,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ee2e57c9cf3943369f0cbd0f8d6c333a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ee2e57c9cf3943369f0cbd0f8d6c333a 2024-11-20T05:16:54,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/c93bfa7e9ebe46d1b5a0050b142e06c9 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/c93bfa7e9ebe46d1b5a0050b142e06c9 2024-11-20T05:16:54,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/252a2e8f05b048c5a9c001a3512cc237 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/252a2e8f05b048c5a9c001a3512cc237 2024-11-20T05:16:54,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/7c5c7e7b5912433c8afaa99c4021d335 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/7c5c7e7b5912433c8afaa99c4021d335 2024-11-20T05:16:54,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/15eb049300a0449fbc30279059c574d3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/15eb049300a0449fbc30279059c574d3 2024-11-20T05:16:54,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/3c4d2e0637d643afa82cde51ba05d178 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/3c4d2e0637d643afa82cde51ba05d178 2024-11-20T05:16:54,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/454b596352cc49fb94ffdf48dfb93035 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/454b596352cc49fb94ffdf48dfb93035 2024-11-20T05:16:54,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9fc8e982754849ef988333a49c1b6477 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/9fc8e982754849ef988333a49c1b6477 2024-11-20T05:16:54,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ba1194fde28446c99e4601805619ea88 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ba1194fde28446c99e4601805619ea88 2024-11-20T05:16:54,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7f66696bee3342d09416ef14d6da0c86, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f783077d7d9d4361b5b410ad6b63db87, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8e43c1af460d41089514c04cb295ad99, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7890ac525d9d448487de89a992ebb389, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8754ee2ae9124fe4b4e9d534a01d1e04, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/499f23f98e224f8e8c9a4cbc669923a4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2f0aea35bf1e43d79aaa7eefbc326efb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/08ae19ead04f40449896c662b381a7f1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/e92aa2d5f4cb4b24ab6599ef7dee58c5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4715fbed395b43a68ce6d02fcee8ace2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5945fb08fb96428386ad58e842b3745b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/3a2de489747740f09dac3c80bee7f5cd, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9e844e18447f4ab3b742113d0e993f62, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/ac27ad75cbfc43a08adeefd4091209b0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f1e8833706224bf7b6d69e1dd1e2ceb1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/fa05d930e2a54ebbbff104f3e22e799f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f2745987e524405888f3b81834a980fb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5a0aefdf7ccd4e2ba1518b24a57df4a3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2ae82078da6c4fad917e6af5234ff899, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9952d7ebe46b4ba8a3058d0489664528, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bff2ed51615f44e4a0028f11694d6029, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bb7420cc4559494ca1602556412aa394, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/884e17988a4c4991a795a04fe7721279, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/0fc5224864bc499e9e0ac596e4d62d19, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/c0c6b9af8ff34cb18a2deead2b36aff6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4736206f1593479181082452ce73cfd0] to archive 2024-11-20T05:16:54,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:54,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7f66696bee3342d09416ef14d6da0c86 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7f66696bee3342d09416ef14d6da0c86 2024-11-20T05:16:54,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f783077d7d9d4361b5b410ad6b63db87 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f783077d7d9d4361b5b410ad6b63db87 2024-11-20T05:16:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8e43c1af460d41089514c04cb295ad99 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8e43c1af460d41089514c04cb295ad99 2024-11-20T05:16:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7890ac525d9d448487de89a992ebb389 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/7890ac525d9d448487de89a992ebb389 2024-11-20T05:16:54,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8754ee2ae9124fe4b4e9d534a01d1e04 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/8754ee2ae9124fe4b4e9d534a01d1e04 2024-11-20T05:16:54,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/499f23f98e224f8e8c9a4cbc669923a4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/499f23f98e224f8e8c9a4cbc669923a4 2024-11-20T05:16:54,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2f0aea35bf1e43d79aaa7eefbc326efb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2f0aea35bf1e43d79aaa7eefbc326efb 2024-11-20T05:16:54,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/08ae19ead04f40449896c662b381a7f1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/08ae19ead04f40449896c662b381a7f1 2024-11-20T05:16:54,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/e92aa2d5f4cb4b24ab6599ef7dee58c5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/e92aa2d5f4cb4b24ab6599ef7dee58c5 2024-11-20T05:16:54,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4715fbed395b43a68ce6d02fcee8ace2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4715fbed395b43a68ce6d02fcee8ace2 2024-11-20T05:16:54,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5945fb08fb96428386ad58e842b3745b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5945fb08fb96428386ad58e842b3745b 2024-11-20T05:16:54,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/3a2de489747740f09dac3c80bee7f5cd to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/3a2de489747740f09dac3c80bee7f5cd 2024-11-20T05:16:54,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9e844e18447f4ab3b742113d0e993f62 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9e844e18447f4ab3b742113d0e993f62 2024-11-20T05:16:54,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/ac27ad75cbfc43a08adeefd4091209b0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/ac27ad75cbfc43a08adeefd4091209b0 2024-11-20T05:16:54,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f1e8833706224bf7b6d69e1dd1e2ceb1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f1e8833706224bf7b6d69e1dd1e2ceb1 2024-11-20T05:16:54,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/fa05d930e2a54ebbbff104f3e22e799f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/fa05d930e2a54ebbbff104f3e22e799f 2024-11-20T05:16:54,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f2745987e524405888f3b81834a980fb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/f2745987e524405888f3b81834a980fb 2024-11-20T05:16:54,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5a0aefdf7ccd4e2ba1518b24a57df4a3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5a0aefdf7ccd4e2ba1518b24a57df4a3 2024-11-20T05:16:54,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2ae82078da6c4fad917e6af5234ff899 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/2ae82078da6c4fad917e6af5234ff899 2024-11-20T05:16:54,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9952d7ebe46b4ba8a3058d0489664528 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9952d7ebe46b4ba8a3058d0489664528 2024-11-20T05:16:54,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bff2ed51615f44e4a0028f11694d6029 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bff2ed51615f44e4a0028f11694d6029 2024-11-20T05:16:54,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bb7420cc4559494ca1602556412aa394 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bb7420cc4559494ca1602556412aa394 2024-11-20T05:16:54,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/884e17988a4c4991a795a04fe7721279 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/884e17988a4c4991a795a04fe7721279 2024-11-20T05:16:54,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/0fc5224864bc499e9e0ac596e4d62d19 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/0fc5224864bc499e9e0ac596e4d62d19 2024-11-20T05:16:54,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/c0c6b9af8ff34cb18a2deead2b36aff6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/c0c6b9af8ff34cb18a2deead2b36aff6 2024-11-20T05:16:54,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4736206f1593479181082452ce73cfd0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/4736206f1593479181082452ce73cfd0 2024-11-20T05:16:54,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/69d688ef589c47afa18edd13208bedb1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a621d596c7914220ab7b080f62d1f3dc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b1284c81b46f4a5a8f8df32542c7d0c1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/f223058744964e0ca1458b28d3e5ce5c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/61e6893e5a824bd2a0e281f4b387028b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/8266b8f7c1ad4507869990f5507c8eb5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd2982d1dcec401c964b7274fb14b5e7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/fef48022bf33480e97caa619a58d4983, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/87f0db5f794f4c90996b789f53608dcb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9236a7358fd444098e0e9bab2f366bd5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/822bb76a18474bbeb6496027c4327490, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/0772bd519a344753ac525c77436339d0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd13fc502ca3492d966dfb38d8c1ff73, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/dc6aaf74072e490381e1202888bcdd20, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/c90d18397ac1441baa77e94e6d32cc32, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/d64fb91db11648f1a03d7b0d5cdcf223, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/629bdcc3b44745548f15615cf4f522f0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9cb67a9f7e0348bebaac5b7faaefc22b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a790b93c74e049e8a03f61930f8e7ad6, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4e9fb4d0667d4c7b835131dab89b2e42, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/2cb3da3625e94a53b25c5a2193625e06, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4cc6f090984f405797aec81f590ec3a1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/3e29afde246548efb72315dfd91dd31a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/577bd6141c9c4bd497822b76765f0903, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e2bde45c077d457a8e42ea00557d5760, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b9a1806a871b42d2a2a31bb085339aaa] to archive 2024-11-20T05:16:54,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:16:54,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/69d688ef589c47afa18edd13208bedb1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/69d688ef589c47afa18edd13208bedb1 2024-11-20T05:16:54,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a621d596c7914220ab7b080f62d1f3dc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a621d596c7914220ab7b080f62d1f3dc 2024-11-20T05:16:54,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b1284c81b46f4a5a8f8df32542c7d0c1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b1284c81b46f4a5a8f8df32542c7d0c1 2024-11-20T05:16:54,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/f223058744964e0ca1458b28d3e5ce5c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/f223058744964e0ca1458b28d3e5ce5c 2024-11-20T05:16:54,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/61e6893e5a824bd2a0e281f4b387028b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/61e6893e5a824bd2a0e281f4b387028b 2024-11-20T05:16:54,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/8266b8f7c1ad4507869990f5507c8eb5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/8266b8f7c1ad4507869990f5507c8eb5 2024-11-20T05:16:54,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd2982d1dcec401c964b7274fb14b5e7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd2982d1dcec401c964b7274fb14b5e7 2024-11-20T05:16:54,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/fef48022bf33480e97caa619a58d4983 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/fef48022bf33480e97caa619a58d4983 2024-11-20T05:16:54,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/87f0db5f794f4c90996b789f53608dcb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/87f0db5f794f4c90996b789f53608dcb 2024-11-20T05:16:54,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9236a7358fd444098e0e9bab2f366bd5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9236a7358fd444098e0e9bab2f366bd5 2024-11-20T05:16:54,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/822bb76a18474bbeb6496027c4327490 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/822bb76a18474bbeb6496027c4327490 2024-11-20T05:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T05:16:54,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/0772bd519a344753ac525c77436339d0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/0772bd519a344753ac525c77436339d0 2024-11-20T05:16:54,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd13fc502ca3492d966dfb38d8c1ff73 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/bd13fc502ca3492d966dfb38d8c1ff73 2024-11-20T05:16:54,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/dc6aaf74072e490381e1202888bcdd20 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/dc6aaf74072e490381e1202888bcdd20 2024-11-20T05:16:54,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/c90d18397ac1441baa77e94e6d32cc32 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/c90d18397ac1441baa77e94e6d32cc32 2024-11-20T05:16:54,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/d64fb91db11648f1a03d7b0d5cdcf223 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/d64fb91db11648f1a03d7b0d5cdcf223 2024-11-20T05:16:54,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/629bdcc3b44745548f15615cf4f522f0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/629bdcc3b44745548f15615cf4f522f0 2024-11-20T05:16:54,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9cb67a9f7e0348bebaac5b7faaefc22b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/9cb67a9f7e0348bebaac5b7faaefc22b 2024-11-20T05:16:54,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a790b93c74e049e8a03f61930f8e7ad6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/a790b93c74e049e8a03f61930f8e7ad6 2024-11-20T05:16:54,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4e9fb4d0667d4c7b835131dab89b2e42 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4e9fb4d0667d4c7b835131dab89b2e42 2024-11-20T05:16:54,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/2cb3da3625e94a53b25c5a2193625e06 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/2cb3da3625e94a53b25c5a2193625e06 2024-11-20T05:16:54,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4cc6f090984f405797aec81f590ec3a1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/4cc6f090984f405797aec81f590ec3a1 2024-11-20T05:16:54,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/3e29afde246548efb72315dfd91dd31a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/3e29afde246548efb72315dfd91dd31a 2024-11-20T05:16:54,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/577bd6141c9c4bd497822b76765f0903 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/577bd6141c9c4bd497822b76765f0903 2024-11-20T05:16:54,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e2bde45c077d457a8e42ea00557d5760 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e2bde45c077d457a8e42ea00557d5760 2024-11-20T05:16:54,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b9a1806a871b42d2a2a31bb085339aaa to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b9a1806a871b42d2a2a31bb085339aaa 2024-11-20T05:16:54,529 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-20T05:16:54,530 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. 2024-11-20T05:16:54,530 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for bef9917e0871717b985dc9ca0a53e5ad: Waiting for close lock at 1732079813545Running coprocessor pre-close hooks at 1732079813545Disabling compacts and flushes for region at 1732079813545Disabling writes for close at 1732079813545Obtaining lock to block concurrent updates at 1732079813546 (+1 ms)Preparing flush snapshotting stores in bef9917e0871717b985dc9ca0a53e5ad at 1732079813546Finished memstore snapshotting TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732079813546Flushing stores of TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad. at 1732079813546Flushing bef9917e0871717b985dc9ca0a53e5ad/A: creating writer at 1732079813546Flushing bef9917e0871717b985dc9ca0a53e5ad/A: appending metadata at 1732079813548 (+2 ms)Flushing bef9917e0871717b985dc9ca0a53e5ad/A: closing flushed file at 1732079813548Flushing bef9917e0871717b985dc9ca0a53e5ad/B: creating writer at 1732079813563 (+15 ms)Flushing bef9917e0871717b985dc9ca0a53e5ad/B: appending metadata at 1732079813565 (+2 ms)Flushing bef9917e0871717b985dc9ca0a53e5ad/B: closing flushed file at 1732079813565Flushing bef9917e0871717b985dc9ca0a53e5ad/C: creating writer at 1732079813979 (+414 ms)Flushing bef9917e0871717b985dc9ca0a53e5ad/C: appending metadata at 1732079813981 (+2 ms)Flushing bef9917e0871717b985dc9ca0a53e5ad/C: closing flushed file at 1732079813981Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b6596f4: reopening flushed file at 1732079814438 (+457 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11f37615: reopening flushed file at 1732079814446 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50313ee6: reopening flushed file at 1732079814451 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for bef9917e0871717b985dc9ca0a53e5ad in 911ms, sequenceid=373, compaction requested=true at 1732079814456 (+5 ms)Writing region close event to WAL at 1732079814527 (+71 ms)Running coprocessor post-close hooks at 1732079814530 (+3 ms)Closed at 1732079814530 2024-11-20T05:16:54,531 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:54,532 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=bef9917e0871717b985dc9ca0a53e5ad, regionState=CLOSED 2024-11-20T05:16:54,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:16:54,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-20T05:16:54,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure bef9917e0871717b985dc9ca0a53e5ad, server=51d5d2011a1c,44753,1732079655401 in 1.1410 sec 2024-11-20T05:16:54,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-20T05:16:54,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bef9917e0871717b985dc9ca0a53e5ad, UNASSIGN in 1.1480 sec 2024-11-20T05:16:54,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-20T05:16:54,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-20T05:16:54,540 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079814540"}]},"ts":"1732079814540"} 2024-11-20T05:16:54,541 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T05:16:54,542 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T05:16:54,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-20T05:16:55,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T05:16:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T05:16:55,538 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:55,538 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T05:16:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:55,540 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T05:16:55,541 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:55,543 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:55,546 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/recovered.edits] 2024-11-20T05:16:55,549 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/06534eb2d4cb45c38396a9f86820c758 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/06534eb2d4cb45c38396a9f86820c758 2024-11-20T05:16:55,551 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ade4e29f674b449e9182b84f8ee533c6 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/ade4e29f674b449e9182b84f8ee533c6 2024-11-20T05:16:55,552 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/be9901b7061f442c8b87b3ef125516e7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/A/be9901b7061f442c8b87b3ef125516e7 2024-11-20T05:16:55,555 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5b539b64169445649f84932c09d4333b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/5b539b64169445649f84932c09d4333b 2024-11-20T05:16:55,557 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9a0be5b18bfe42de977962bf7a24776c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/9a0be5b18bfe42de977962bf7a24776c 2024-11-20T05:16:55,558 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bfbeabf0b04a428caed22a2d00959a62 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/B/bfbeabf0b04a428caed22a2d00959a62 2024-11-20T05:16:55,561 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/44c6f330c37e4eaabed15bb0fb562f5a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/44c6f330c37e4eaabed15bb0fb562f5a 2024-11-20T05:16:55,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b97475fddafe459b8eff9936c46d93fe to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/b97475fddafe459b8eff9936c46d93fe 2024-11-20T05:16:55,563 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e4ce2d81b6d344ef8753312e73e48671 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/C/e4ce2d81b6d344ef8753312e73e48671 2024-11-20T05:16:55,567 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/recovered.edits/376.seqid to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad/recovered.edits/376.seqid 2024-11-20T05:16:55,567 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/bef9917e0871717b985dc9ca0a53e5ad 2024-11-20T05:16:55,567 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T05:16:55,570 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:55,573 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T05:16:55,576 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T05:16:55,577 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:55,577 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T05:16:55,577 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732079815577"}]},"ts":"9223372036854775807"} 2024-11-20T05:16:55,579 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T05:16:55,580 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bef9917e0871717b985dc9ca0a53e5ad, NAME => 'TestAcidGuarantees,,1732079789139.bef9917e0871717b985dc9ca0a53e5ad.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T05:16:55,580 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T05:16:55,580 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732079815580"}]},"ts":"9223372036854775807"} 2024-11-20T05:16:55,582 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T05:16:55,583 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:55,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-20T05:16:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T05:16:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T05:16:55,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:55,664 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=421 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=4553 (was 4583) 2024-11-20T05:16:55,672 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=421, ProcessCount=11, AvailableMemoryMB=4552 2024-11-20T05:16:55,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:16:55,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:55,675 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T05:16:55,675 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:55,675 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-20T05:16:55,676 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T05:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-20T05:16:55,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742321_1497 (size=1061) 2024-11-20T05:16:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-20T05:16:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-20T05:16:56,086 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673 2024-11-20T05:16:56,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742322_1498 (size=53) 2024-11-20T05:16:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-20T05:16:56,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:16:56,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee51cd6a06f90dcb79cbe1ee3eb2fbd4, disabling compactions & flushes 2024-11-20T05:16:56,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. after waiting 0 ms 2024-11-20T05:16:56,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: Waiting for close lock at 1732079816496Disabling compacts and flushes for region at 1732079816496Disabling writes for close at 1732079816496Writing region close event to WAL at 1732079816496Closed at 1732079816496 2024-11-20T05:16:56,498 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T05:16:56,498 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732079816498"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732079816498"}]},"ts":"1732079816498"} 2024-11-20T05:16:56,499 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T05:16:56,500 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T05:16:56,500 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079816500"}]},"ts":"1732079816500"} 2024-11-20T05:16:56,502 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T05:16:56,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, ASSIGN}] 2024-11-20T05:16:56,503 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, ASSIGN 2024-11-20T05:16:56,504 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, ASSIGN; state=OFFLINE, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=false 2024-11-20T05:16:56,655 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:56,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, ASSIGN because future has completed 2024-11-20T05:16:56,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:16:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-20T05:16:56,818 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,818 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:16:56,819 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,819 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:16:56,820 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,820 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,822 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,824 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:56,824 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 columnFamilyName A 2024-11-20T05:16:56,824 DEBUG [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:56,825 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(327): Store=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:56,825 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,826 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:56,827 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 columnFamilyName B 2024-11-20T05:16:56,827 DEBUG [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:56,827 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(327): Store=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:56,827 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,829 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:56,829 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 columnFamilyName C 2024-11-20T05:16:56,829 DEBUG [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:56,830 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(327): Store=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:56,830 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,830 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,831 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,831 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,832 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,832 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,833 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:16:56,834 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,837 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T05:16:56,838 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened ee51cd6a06f90dcb79cbe1ee3eb2fbd4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67875162, jitterRate=0.011418730020523071}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:16:56,838 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:56,839 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: Running coprocessor pre-open hook at 1732079816820Writing region info on filesystem at 1732079816820Initializing all the Stores at 1732079816821 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079816821Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079816822 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079816822Cleaning up temporary data from old regions at 1732079816832 (+10 ms)Running coprocessor post-open hooks at 1732079816838 (+6 ms)Region opened successfully at 1732079816839 (+1 ms) 2024-11-20T05:16:56,840 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., pid=134, masterSystemTime=1732079816812 2024-11-20T05:16:56,841 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,841 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:56,842 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=OPEN, openSeqNum=2, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:56,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:16:56,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T05:16:56,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 in 185 msec 2024-11-20T05:16:56,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-20T05:16:56,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, ASSIGN in 343 msec 2024-11-20T05:16:56,847 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T05:16:56,847 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079816847"}]},"ts":"1732079816847"} 2024-11-20T05:16:56,848 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T05:16:56,849 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T05:16:56,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-20T05:16:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-20T05:16:57,818 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:57,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eedb90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:57,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:57,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:57,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc93e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:57,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c118e3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:57,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:57,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:57,832 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T05:16:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 51d5d2011a1c,37927,1732079654666 2024-11-20T05:16:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@cd2fb74 2024-11-20T05:16:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T05:16:57,833 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T05:16:57,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T05:16:57,834 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T05:16:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T05:16:57,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742323_1499 (size=1097) 2024-11-20T05:16:58,248 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T05:16:58,249 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T05:16:58,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:16:58,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, REOPEN/MOVE}] 2024-11-20T05:16:58,257 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, REOPEN/MOVE 2024-11-20T05:16:58,257 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, REOPEN/MOVE because future has completed 2024-11-20T05:16:58,259 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:16:58,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:16:58,412 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,412 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:16:58,413 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing ee51cd6a06f90dcb79cbe1ee3eb2fbd4, disabling compactions & flushes 2024-11-20T05:16:58,413 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,413 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,413 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. after waiting 0 ms 2024-11-20T05:16:58,413 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,423 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T05:16:58,424 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,424 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: Waiting for close lock at 1732079818412Running coprocessor pre-close hooks at 1732079818412Disabling compacts and flushes for region at 1732079818412Disabling writes for close at 1732079818413 (+1 ms)Writing region close event to WAL at 1732079818414 (+1 ms)Running coprocessor post-close hooks at 1732079818424 (+10 ms)Closed at 1732079818424 2024-11-20T05:16:58,424 WARN [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to self. 2024-11-20T05:16:58,426 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=CLOSED 2024-11-20T05:16:58,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:16:58,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T05:16:58,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 in 170 msec 2024-11-20T05:16:58,431 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, REOPEN/MOVE; state=CLOSED, location=51d5d2011a1c,44753,1732079655401; forceNewPlan=false, retain=true 2024-11-20T05:16:58,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=OPENING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, REOPEN/MOVE because future has completed 2024-11-20T05:16:58,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:16:58,749 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,749 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T05:16:58,750 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,751 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T05:16:58,751 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,751 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,755 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,755 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:58,755 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 columnFamilyName A 2024-11-20T05:16:58,756 DEBUG [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:58,757 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(327): Store=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:58,757 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,758 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:58,758 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 columnFamilyName B 2024-11-20T05:16:58,758 DEBUG [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:58,758 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(327): Store=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:58,758 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,759 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T05:16:58,759 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee51cd6a06f90dcb79cbe1ee3eb2fbd4 columnFamilyName C 2024-11-20T05:16:58,759 DEBUG [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:58,759 INFO [StoreOpener-ee51cd6a06f90dcb79cbe1ee3eb2fbd4-1 {}] regionserver.HStore(327): Store=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T05:16:58,759 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,759 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,760 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,761 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,761 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,761 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,762 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T05:16:58,763 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,763 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened ee51cd6a06f90dcb79cbe1ee3eb2fbd4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63968056, jitterRate=-0.04680168628692627}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T05:16:58,764 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,764 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: Running coprocessor pre-open hook at 1732079818752Writing region info on filesystem at 1732079818752Initializing all the Stores at 1732079818754 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079818754Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079818754Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732079818754Cleaning up temporary data from old regions at 1732079818761 (+7 ms)Running coprocessor post-open hooks at 1732079818764 (+3 ms)Region opened successfully at 1732079818764 2024-11-20T05:16:58,765 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., pid=139, masterSystemTime=1732079818741 2024-11-20T05:16:58,766 DEBUG [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,766 INFO [RS_OPEN_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,767 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=OPEN, openSeqNum=5, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:16:58,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-20T05:16:58,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 in 181 msec 2024-11-20T05:16:58,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-20T05:16:58,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, REOPEN/MOVE in 514 msec 2024-11-20T05:16:58,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-20T05:16:58,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-20T05:16:58,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-20T05:16:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T05:16:58,775 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:16:58,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bc30a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,777 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c54d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,778 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739dbef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876a305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,780 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7d96fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,781 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76ae4b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6266891e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,782 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d6bd174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393b4e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79538ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,785 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e29703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdf05bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f076f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,788 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431f4dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,789 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d11ff74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22935e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,791 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba44e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,792 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109b377e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc7876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,794 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b702f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,795 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427d21e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e72bfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,797 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c60e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,799 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c84c60a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5a171f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,801 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54323884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,802 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb01e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7278e824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51d5d2011a1c,37927,-1 for getting cluster id 2024-11-20T05:16:58,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T05:16:58,804 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b9515f8-070c-401f-827a-a94aa4386eb6' 2024-11-20T05:16:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T05:16:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b9515f8-070c-401f-827a-a94aa4386eb6" 2024-11-20T05:16:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190dae9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51d5d2011a1c,37927,-1] 2024-11-20T05:16:58,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T05:16:58,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:16:58,805 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T05:16:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd29804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T05:16:58,806 DEBUG [Thread-2257 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,807 DEBUG [Thread-2259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,807 DEBUG [Thread-2250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,807 DEBUG [Thread-2246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,807 DEBUG [Thread-2261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,807 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,807 DEBUG [Thread-2252 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:16:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,809 DEBUG [Thread-2254 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,809 DEBUG [Thread-2248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-20T05:16:58,810 DEBUG [Thread-2254 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,810 DEBUG [Thread-2254 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-20T05:16:58,812 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:16:58,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,812 DEBUG [Thread-2263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T05:16:58,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51d5d2011a1c,44753,1732079655401, seqNum=-1] 2024-11-20T05:16:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T05:16:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,814 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:16:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T05:16:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:16:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5] 2024-11-20T05:16:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:16:58,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:16:58,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:58,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:16:58,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:58,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:16:58,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:58,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079818834, max timestamp=1732079818845Num uniques 859; 2024-11-20T05:16:58,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:16:58,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079818834, max timestamp=1732079818845Num uniques 859; 2024-11-20T05:16:58,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200aecedd433e84b5b9b63e00d945c5dee_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079818820/Put/seqid=0 2024-11-20T05:16:58,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:16:58,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079818834, max timestamp=1732079818845Num uniques 859; 2024-11-20T05:16:58,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:16:58,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742324_1500 (size=12154) 2024-11-20T05:16:58,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:58,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079878868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,875 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200aecedd433e84b5b9b63e00d945c5dee_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200aecedd433e84b5b9b63e00d945c5dee_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079878869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079878869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079878869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079878870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,877 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/7368636bdfea4253aa9e8f3dc287471b, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:16:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:58,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/7368636bdfea4253aa9e8f3dc287471b is 175, key is test_row_0/A:col10/1732079818820/Put/seqid=0 2024-11-20T05:16:58,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742325_1501 (size=30955) 2024-11-20T05:16:58,888 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/7368636bdfea4253aa9e8f3dc287471b 2024-11-20T05:16:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-20T05:16:58,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/58472683320d4a31970667da0fdc6f31 is 50, key is test_row_0/B:col10/1732079818820/Put/seqid=0 2024-11-20T05:16:58,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742326_1502 (size=12001) 2024-11-20T05:16:58,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/58472683320d4a31970667da0fdc6f31 2024-11-20T05:16:58,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8000049651aa424396db5bb6c4177139 is 50, key is test_row_0/C:col10/1732079818820/Put/seqid=0 2024-11-20T05:16:58,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742327_1503 (size=12001) 2024-11-20T05:16:58,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8000049651aa424396db5bb6c4177139 2024-11-20T05:16:58,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-20T05:16:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:16:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:58,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/7368636bdfea4253aa9e8f3dc287471b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/7368636bdfea4253aa9e8f3dc287471b 2024-11-20T05:16:58,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/7368636bdfea4253aa9e8f3dc287471b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-20T05:16:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/58472683320d4a31970667da0fdc6f31 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/58472683320d4a31970667da0fdc6f31 2024-11-20T05:16:58,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/58472683320d4a31970667da0fdc6f31, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T05:16:58,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8000049651aa424396db5bb6c4177139 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8000049651aa424396db5bb6c4177139 2024-11-20T05:16:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079878976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:58,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8000049651aa424396db5bb6c4177139, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T05:16:58,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 166ms, sequenceid=16, compaction requested=false 2024-11-20T05:16:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:16:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:58,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T05:16:58,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:16:58,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:58,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:16:58,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:58,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:16:58,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:16:59,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112000ab3f118bb542a180dd67ba31057be8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079818864/Put/seqid=0 2024-11-20T05:16:59,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079879000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742328_1504 (size=17034) 2024-11-20T05:16:59,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079879002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079879002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079879004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112000ab3f118bb542a180dd67ba31057be8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000ab3f118bb542a180dd67ba31057be8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:16:59,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9101b9f866d74b869aceb5c6998ca692, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:16:59,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9101b9f866d74b869aceb5c6998ca692 is 175, key is test_row_0/A:col10/1732079818864/Put/seqid=0 2024-11-20T05:16:59,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742329_1505 (size=48139) 2024-11-20T05:16:59,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9101b9f866d74b869aceb5c6998ca692 2024-11-20T05:16:59,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/8991d25e4ba440b6a7273667b9ec4ae7 is 50, key is test_row_0/B:col10/1732079818864/Put/seqid=0 2024-11-20T05:16:59,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742330_1506 (size=12001) 2024-11-20T05:16:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079879117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079879117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079879118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079879118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-20T05:16:59,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:16:59,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-20T05:16:59,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079879187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-20T05:16:59,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:16:59,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079879328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079879328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079879328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079879328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/8991d25e4ba440b6a7273667b9ec4ae7 2024-11-20T05:16:59,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-20T05:16:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:16:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-20T05:16:59,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/1eda432395534b5fb52846cdf058b8a4 is 50, key is test_row_0/C:col10/1732079818864/Put/seqid=0 2024-11-20T05:16:59,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742331_1507 (size=12001) 2024-11-20T05:16:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079879497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-20T05:16:59,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:16:59,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079879637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079879637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079879638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079879638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:16:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:16:59,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-20T05:16:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:16:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:16:59,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/1eda432395534b5fb52846cdf058b8a4 2024-11-20T05:16:59,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9101b9f866d74b869aceb5c6998ca692 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9101b9f866d74b869aceb5c6998ca692 2024-11-20T05:16:59,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9101b9f866d74b869aceb5c6998ca692, entries=250, sequenceid=53, filesize=47.0 K 2024-11-20T05:16:59,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/8991d25e4ba440b6a7273667b9ec4ae7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8991d25e4ba440b6a7273667b9ec4ae7 2024-11-20T05:16:59,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8991d25e4ba440b6a7273667b9ec4ae7, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T05:16:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/1eda432395534b5fb52846cdf058b8a4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/1eda432395534b5fb52846cdf058b8a4 2024-11-20T05:16:59,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/1eda432395534b5fb52846cdf058b8a4, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T05:16:59,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 896ms, sequenceid=53, compaction requested=false 2024-11-20T05:16:59,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:16:59,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-20T05:16:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:16:59,926 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T05:16:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:16:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:16:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:16:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:16:59,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206141645ada9240e4af1ae14a769177d1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079818995/Put/seqid=0 2024-11-20T05:16:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-20T05:16:59,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742332_1508 (size=12154) 2024-11-20T05:17:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:00,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:00,053 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079820059, max timestamp=1732079820086Num uniques 859; 2024-11-20T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079820059, max timestamp=1732079820086Num uniques 859; 2024-11-20T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:00,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079820059, max timestamp=1732079820086Num uniques 859; 2024-11-20T05:17:00,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079880151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079880152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079880153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079880154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079820150, max timestamp=1732079820158Num uniques 1018; 2024-11-20T05:17:00,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079820150, max timestamp=1732079820158Num uniques 1018; 2024-11-20T05:17:00,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:00,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079820150, max timestamp=1732079820158Num uniques 1018; 2024-11-20T05:17:00,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:00,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079820227, max timestamp=1732079820255Num uniques 1027; 2024-11-20T05:17:00,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:00,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079820227, max timestamp=1732079820255Num uniques 1027; 2024-11-20T05:17:00,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079820227, max timestamp=1732079820255Num uniques 1027; 2024-11-20T05:17:00,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:00,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079880283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079880286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732079820274, max timestamp=1732079820289Num uniques 1036; 2024-11-20T05:17:00,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-20T05:17:00,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732079820274, max timestamp=1732079820289Num uniques 1036; 2024-11-20T05:17:00,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-20T05:17:00,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732079820274, max timestamp=1732079820289Num uniques 1036; 2024-11-20T05:17:00,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-20T05:17:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079880316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079880318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079880319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:00,354 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206141645ada9240e4af1ae14a769177d1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206141645ada9240e4af1ae14a769177d1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:00,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/952742d697804ba4b0adf2de0e7013cc, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/952742d697804ba4b0adf2de0e7013cc is 175, key is test_row_0/A:col10/1732079818995/Put/seqid=0 2024-11-20T05:17:00,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742333_1509 (size=30955) 2024-11-20T05:17:00,358 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/952742d697804ba4b0adf2de0e7013cc 2024-11-20T05:17:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/07a3b3a8adf34737a1afbdaff24bd7b8 is 50, key is test_row_0/B:col10/1732079818995/Put/seqid=0 2024-11-20T05:17:00,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742334_1510 (size=12001) 2024-11-20T05:17:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079880397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079880397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079880427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079880427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079880427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079880607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079880607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079880638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079880638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079880638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,766 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/07a3b3a8adf34737a1afbdaff24bd7b8 2024-11-20T05:17:00,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/af8517fe427e429fb0f56c9312f1e229 is 50, key is test_row_0/C:col10/1732079818995/Put/seqid=0 2024-11-20T05:17:00,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742335_1511 (size=12001) 2024-11-20T05:17:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079880917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079880917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079880947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079880947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079880947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-20T05:17:01,174 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/af8517fe427e429fb0f56c9312f1e229 2024-11-20T05:17:01,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/952742d697804ba4b0adf2de0e7013cc as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/952742d697804ba4b0adf2de0e7013cc 2024-11-20T05:17:01,180 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/952742d697804ba4b0adf2de0e7013cc, entries=150, sequenceid=62, filesize=30.2 K 2024-11-20T05:17:01,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/07a3b3a8adf34737a1afbdaff24bd7b8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/07a3b3a8adf34737a1afbdaff24bd7b8 2024-11-20T05:17:01,183 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/07a3b3a8adf34737a1afbdaff24bd7b8, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T05:17:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/af8517fe427e429fb0f56c9312f1e229 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/af8517fe427e429fb0f56c9312f1e229 2024-11-20T05:17:01,187 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/af8517fe427e429fb0f56c9312f1e229, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T05:17:01,187 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1261ms, sequenceid=62, compaction requested=true 2024-11-20T05:17:01,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:01,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:01,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-20T05:17:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-20T05:17:01,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-20T05:17:01,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3740 sec 2024-11-20T05:17:01,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.3820 sec 2024-11-20T05:17:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:01,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=147.60 KB heapSize=412.92 KB 2024-11-20T05:17:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T05:17:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T05:17:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T05:17:01,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112019de7e60d3a249baba421d5ee505e9a1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079820317/Put/seqid=0 2024-11-20T05:17:01,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742336_1512 (size=14794) 2024-11-20T05:17:01,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079881455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079881456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079881457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079881457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079881457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079881567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079881567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079881777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079881777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:01,859 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:01,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112019de7e60d3a249baba421d5ee505e9a1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112019de7e60d3a249baba421d5ee505e9a1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:01,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d4b313108a79496396298f1505479e2a, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:01,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d4b313108a79496396298f1505479e2a is 175, key is test_row_0/A:col10/1732079820317/Put/seqid=0 2024-11-20T05:17:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742337_1513 (size=39749) 2024-11-20T05:17:01,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d4b313108a79496396298f1505479e2a 2024-11-20T05:17:01,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/4ce9abc6b2904293a3287905ce1d47bf is 50, key is test_row_0/B:col10/1732079820317/Put/seqid=0 2024-11-20T05:17:01,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742338_1514 (size=12151) 2024-11-20T05:17:01,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/4ce9abc6b2904293a3287905ce1d47bf 2024-11-20T05:17:01,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/79a30cf094f44f02be8d91371724c5e0 is 50, key is test_row_0/C:col10/1732079820317/Put/seqid=0 2024-11-20T05:17:01,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742339_1515 (size=12151) 2024-11-20T05:17:01,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/79a30cf094f44f02be8d91371724c5e0 2024-11-20T05:17:01,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d4b313108a79496396298f1505479e2a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d4b313108a79496396298f1505479e2a 2024-11-20T05:17:01,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d4b313108a79496396298f1505479e2a, entries=200, sequenceid=151, filesize=38.8 K 2024-11-20T05:17:01,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/4ce9abc6b2904293a3287905ce1d47bf as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/4ce9abc6b2904293a3287905ce1d47bf 2024-11-20T05:17:01,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/4ce9abc6b2904293a3287905ce1d47bf, entries=150, sequenceid=151, filesize=11.9 K 2024-11-20T05:17:01,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/79a30cf094f44f02be8d91371724c5e0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/79a30cf094f44f02be8d91371724c5e0 2024-11-20T05:17:01,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/79a30cf094f44f02be8d91371724c5e0, entries=150, sequenceid=151, filesize=11.9 K 2024-11-20T05:17:01,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~433.50 KB/443904, currentSize=26.84 KB/27480 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 485ms, sequenceid=151, compaction requested=true 2024-11-20T05:17:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:17:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:17:01,926 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:17:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:17:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:01,926 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A is initiating minor compaction (all files) 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B is initiating minor compaction (all files) 2024-11-20T05:17:01,927 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:01,927 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:01,927 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/58472683320d4a31970667da0fdc6f31, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8991d25e4ba440b6a7273667b9ec4ae7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/07a3b3a8adf34737a1afbdaff24bd7b8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/4ce9abc6b2904293a3287905ce1d47bf] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=47.0 K 2024-11-20T05:17:01,927 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/7368636bdfea4253aa9e8f3dc287471b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9101b9f866d74b869aceb5c6998ca692, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/952742d697804ba4b0adf2de0e7013cc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d4b313108a79496396298f1505479e2a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=146.3 K 2024-11-20T05:17:01,927 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/7368636bdfea4253aa9e8f3dc287471b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9101b9f866d74b869aceb5c6998ca692, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/952742d697804ba4b0adf2de0e7013cc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d4b313108a79496396298f1505479e2a] 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 58472683320d4a31970667da0fdc6f31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732079818819 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7368636bdfea4253aa9e8f3dc287471b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732079818819 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8991d25e4ba440b6a7273667b9ec4ae7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732079818864 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9101b9f866d74b869aceb5c6998ca692, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732079818864 2024-11-20T05:17:01,927 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a3b3a8adf34737a1afbdaff24bd7b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079818995 2024-11-20T05:17:01,928 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce9abc6b2904293a3287905ce1d47bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732079820317 2024-11-20T05:17:01,928 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 952742d697804ba4b0adf2de0e7013cc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079818995 2024-11-20T05:17:01,929 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b313108a79496396298f1505479e2a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732079820317 2024-11-20T05:17:01,937 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:01,943 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:01,944 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/de818a63b80f4eba9faf12d0d184ea49 is 50, key is test_row_0/B:col10/1732079820317/Put/seqid=0 2024-11-20T05:17:01,949 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411200ba86b780a304414a6babdda0681395b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:01,952 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411200ba86b780a304414a6babdda0681395b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:01,952 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200ba86b780a304414a6babdda0681395b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:01,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742340_1516 (size=12289) 2024-11-20T05:17:01,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742341_1517 (size=4469) 2024-11-20T05:17:01,963 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#A#compaction#437 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:01,964 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/b5209ec6d86a48bdad2b536da04def9c is 175, key is test_row_0/A:col10/1732079820317/Put/seqid=0 2024-11-20T05:17:01,966 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/de818a63b80f4eba9faf12d0d184ea49 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de818a63b80f4eba9faf12d0d184ea49 2024-11-20T05:17:01,989 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into de818a63b80f4eba9faf12d0d184ea49(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:01,989 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:01,989 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, priority=12, startTime=1732079821926; duration=0sec 2024-11-20T05:17:01,989 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:01,989 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B 2024-11-20T05:17:01,990 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:17:01,991 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:17:01,991 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C is initiating minor compaction (all files) 2024-11-20T05:17:01,991 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:01,991 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8000049651aa424396db5bb6c4177139, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/1eda432395534b5fb52846cdf058b8a4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/af8517fe427e429fb0f56c9312f1e229, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/79a30cf094f44f02be8d91371724c5e0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=47.0 K 2024-11-20T05:17:01,992 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8000049651aa424396db5bb6c4177139, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732079818819 2024-11-20T05:17:01,992 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eda432395534b5fb52846cdf058b8a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732079818864 2024-11-20T05:17:01,993 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting af8517fe427e429fb0f56c9312f1e229, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732079818995 2024-11-20T05:17:01,993 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a30cf094f44f02be8d91371724c5e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732079820317 2024-11-20T05:17:02,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742342_1518 (size=31243) 2024-11-20T05:17:02,022 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:02,022 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/0a0ddb32b5e14e5eb16e9239de0c7df5 is 50, key is test_row_0/C:col10/1732079820317/Put/seqid=0 2024-11-20T05:17:02,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742343_1519 (size=12289) 2024-11-20T05:17:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:02,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:17:02,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:02,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:02,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:02,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:02,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:02,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:02,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206ded9f44367a44e8b157bd8924e00cfc_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079822090/Put/seqid=0 2024-11-20T05:17:02,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742344_1520 (size=12304) 2024-11-20T05:17:02,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:02,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206ded9f44367a44e8b157bd8924e00cfc_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206ded9f44367a44e8b157bd8924e00cfc_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:02,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a2224a0c332e4a20a826d742caa3d629, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:02,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a2224a0c332e4a20a826d742caa3d629 is 175, key is test_row_0/A:col10/1732079822090/Put/seqid=0 2024-11-20T05:17:02,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742345_1521 (size=31105) 2024-11-20T05:17:02,108 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a2224a0c332e4a20a826d742caa3d629 2024-11-20T05:17:02,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/abaebafeacb64315b0c1ffdb122d24bb is 50, key is test_row_0/B:col10/1732079822090/Put/seqid=0 2024-11-20T05:17:02,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742346_1522 (size=12151) 2024-11-20T05:17:02,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079822122, max timestamp=1732079822137Num uniques 859; 2024-11-20T05:17:02,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079822122, max timestamp=1732079822137Num uniques 859; 2024-11-20T05:17:02,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:02,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:02,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079822122, max timestamp=1732079822137Num uniques 859; 2024-11-20T05:17:02,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:02,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079882184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079882185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079882297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079882297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,412 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/b5209ec6d86a48bdad2b536da04def9c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/b5209ec6d86a48bdad2b536da04def9c 2024-11-20T05:17:02,415 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into b5209ec6d86a48bdad2b536da04def9c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:02,416 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:02,416 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, priority=12, startTime=1732079821926; duration=0sec 2024-11-20T05:17:02,416 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:02,416 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A 2024-11-20T05:17:02,441 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/0a0ddb32b5e14e5eb16e9239de0c7df5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0a0ddb32b5e14e5eb16e9239de0c7df5 2024-11-20T05:17:02,444 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 0a0ddb32b5e14e5eb16e9239de0c7df5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:02,444 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:02,445 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, priority=12, startTime=1732079821926; duration=0sec 2024-11-20T05:17:02,445 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:02,445 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C 2024-11-20T05:17:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079882477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079882477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079882477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,479 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:02,479 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:02,479 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079882507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079882508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/abaebafeacb64315b0c1ffdb122d24bb 2024-11-20T05:17:02,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/6c14e24161d54a0db44587610cd95f0c is 50, key is test_row_0/C:col10/1732079822090/Put/seqid=0 2024-11-20T05:17:02,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742347_1523 (size=12151) 2024-11-20T05:17:02,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079882817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079882817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:02,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/6c14e24161d54a0db44587610cd95f0c 2024-11-20T05:17:02,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a2224a0c332e4a20a826d742caa3d629 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a2224a0c332e4a20a826d742caa3d629 2024-11-20T05:17:02,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a2224a0c332e4a20a826d742caa3d629, entries=150, sequenceid=162, filesize=30.4 K 2024-11-20T05:17:02,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/abaebafeacb64315b0c1ffdb122d24bb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/abaebafeacb64315b0c1ffdb122d24bb 2024-11-20T05:17:02,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/abaebafeacb64315b0c1ffdb122d24bb, entries=150, sequenceid=162, filesize=11.9 K 2024-11-20T05:17:02,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/6c14e24161d54a0db44587610cd95f0c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/6c14e24161d54a0db44587610cd95f0c 2024-11-20T05:17:02,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/6c14e24161d54a0db44587610cd95f0c, entries=150, sequenceid=162, filesize=11.9 K 2024-11-20T05:17:02,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 859ms, sequenceid=162, compaction requested=false 2024-11-20T05:17:02,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-20T05:17:02,978 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:02,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-20T05:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T05:17:02,980 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:17:02,981 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:17:02,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:17:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T05:17:03,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-20T05:17:03,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:03,133 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T05:17:03,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:03,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:03,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:03,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:03,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:03,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:03,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cb1edd834d8f4b03ab4d36b4337715b8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079822184/Put/seqid=0 2024-11-20T05:17:03,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742348_1524 (size=12304) 2024-11-20T05:17:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T05:17:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:03,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079883342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079883347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:03,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079883457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079883457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:03,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:03,544 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cb1edd834d8f4b03ab4d36b4337715b8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb1edd834d8f4b03ab4d36b4337715b8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:03,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cc0db878a76e4f5dbd6332098e7e97c0, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cc0db878a76e4f5dbd6332098e7e97c0 is 175, key is test_row_0/A:col10/1732079822184/Put/seqid=0 2024-11-20T05:17:03,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742349_1525 (size=31105) 2024-11-20T05:17:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T05:17:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079883667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079883667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:03,949 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cc0db878a76e4f5dbd6332098e7e97c0 2024-11-20T05:17:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/e5e8d5aff3224e20a25fdbff8e1be534 is 50, key is test_row_0/B:col10/1732079822184/Put/seqid=0 2024-11-20T05:17:03,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742350_1526 (size=12151) 2024-11-20T05:17:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079883977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079883978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T05:17:04,357 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/e5e8d5aff3224e20a25fdbff8e1be534 2024-11-20T05:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/e9d228c62cd247b995037ff699cef550 is 50, key is test_row_0/C:col10/1732079822184/Put/seqid=0 2024-11-20T05:17:04,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742351_1527 (size=12151) 2024-11-20T05:17:04,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079884488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:04,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079884488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:04,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079884488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:04,491 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079884497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079884497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:04,499 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,499 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:04,766 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/e9d228c62cd247b995037ff699cef550 2024-11-20T05:17:04,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cc0db878a76e4f5dbd6332098e7e97c0 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cc0db878a76e4f5dbd6332098e7e97c0 2024-11-20T05:17:04,773 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cc0db878a76e4f5dbd6332098e7e97c0, entries=150, sequenceid=201, filesize=30.4 K 2024-11-20T05:17:04,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/e5e8d5aff3224e20a25fdbff8e1be534 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/e5e8d5aff3224e20a25fdbff8e1be534 2024-11-20T05:17:04,777 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/e5e8d5aff3224e20a25fdbff8e1be534, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T05:17:04,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/e9d228c62cd247b995037ff699cef550 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e9d228c62cd247b995037ff699cef550 2024-11-20T05:17:04,780 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e9d228c62cd247b995037ff699cef550, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T05:17:04,781 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1648ms, sequenceid=201, compaction requested=true 2024-11-20T05:17:04,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-20T05:17:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-20T05:17:04,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-20T05:17:04,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-20T05:17:04,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.8070 sec 2024-11-20T05:17:04,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T05:17:04,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T05:17:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T05:17:05,127 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:05,127 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:17:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-20T05:17:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T05:17:05,129 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:17:05,129 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:17:05,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:17:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T05:17:05,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T05:17:05,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:05,281 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T05:17:05,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:05,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:05,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:05,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:05,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200feb17577d8d425b8f44472c336ec74b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079823346/Put/seqid=0 2024-11-20T05:17:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742352_1528 (size=12304) 2024-11-20T05:17:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T05:17:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:05,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:05,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079825528, max timestamp=1732079825546Num uniques 859; 2024-11-20T05:17:05,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:05,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079825528, max timestamp=1732079825546Num uniques 859; 2024-11-20T05:17:05,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:05,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079825528, max timestamp=1732079825546Num uniques 859; 2024-11-20T05:17:05,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:05,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079885595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:05,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079885600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:05,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:05,695 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200feb17577d8d425b8f44472c336ec74b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200feb17577d8d425b8f44472c336ec74b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/45f93d78acc84b9390e8e43abb5dd2be, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/45f93d78acc84b9390e8e43abb5dd2be is 175, key is test_row_0/A:col10/1732079823346/Put/seqid=0 2024-11-20T05:17:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742353_1529 (size=31105) 2024-11-20T05:17:05,699 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/45f93d78acc84b9390e8e43abb5dd2be 2024-11-20T05:17:05,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/f4754598a790409e8e588f7d0daa183f is 50, key is test_row_0/B:col10/1732079823346/Put/seqid=0 2024-11-20T05:17:05,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742354_1530 (size=12151) 2024-11-20T05:17:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079885707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:05,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079885707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T05:17:05,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079885918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079885918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:06,107 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/f4754598a790409e8e588f7d0daa183f 2024-11-20T05:17:06,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/d0766cbbe2eb4c0e80a8934b2f9e4367 is 50, key is test_row_0/C:col10/1732079823346/Put/seqid=0 2024-11-20T05:17:06,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742355_1531 (size=12151) 2024-11-20T05:17:06,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079886227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:06,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079886227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T05:17:06,522 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/d0766cbbe2eb4c0e80a8934b2f9e4367 2024-11-20T05:17:06,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/45f93d78acc84b9390e8e43abb5dd2be as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/45f93d78acc84b9390e8e43abb5dd2be 2024-11-20T05:17:06,529 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/45f93d78acc84b9390e8e43abb5dd2be, entries=150, sequenceid=211, filesize=30.4 K 2024-11-20T05:17:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/f4754598a790409e8e588f7d0daa183f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4754598a790409e8e588f7d0daa183f 2024-11-20T05:17:06,532 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4754598a790409e8e588f7d0daa183f, entries=150, sequenceid=211, filesize=11.9 K 2024-11-20T05:17:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/d0766cbbe2eb4c0e80a8934b2f9e4367 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d0766cbbe2eb4c0e80a8934b2f9e4367 2024-11-20T05:17:06,535 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d0766cbbe2eb4c0e80a8934b2f9e4367, entries=150, sequenceid=211, filesize=11.9 K 2024-11-20T05:17:06,535 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1254ms, sequenceid=211, compaction requested=true 2024-11-20T05:17:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-20T05:17:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-20T05:17:06,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-20T05:17:06,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-20T05:17:06,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.4120 sec 2024-11-20T05:17:06,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079825580, max timestamp=1732079826741Num uniques 1018; 2024-11-20T05:17:06,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:06,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079825580, max timestamp=1732079826741Num uniques 1018; 2024-11-20T05:17:06,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:06,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-20T05:17:06,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079825580, max timestamp=1732079826741Num uniques 1018; 2024-11-20T05:17:06,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:06,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:06,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:06,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:06,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e0da5f46f61f40488495b236e06b25c2_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079825592/Put/seqid=0 2024-11-20T05:17:06,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742356_1532 (size=14794) 2024-11-20T05:17:06,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079826791, max timestamp=1732079826801Num uniques 859; 2024-11-20T05:17:06,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:06,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079826791, max timestamp=1732079826801Num uniques 859; 2024-11-20T05:17:06,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079826791, max timestamp=1732079826801Num uniques 859; 2024-11-20T05:17:06,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:06,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:06,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079826832, max timestamp=1732079826846Num uniques 1018; 2024-11-20T05:17:06,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:06,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079826832, max timestamp=1732079826846Num uniques 1018; 2024-11-20T05:17:06,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:06,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079826832, max timestamp=1732079826846Num uniques 1018; 2024-11-20T05:17:06,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T05:17:06,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079826882, max timestamp=1732079826893Num uniques 1027; 2024-11-20T05:17:06,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079826882, max timestamp=1732079826893Num uniques 1027; 2024-11-20T05:17:06,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:06,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:06,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079826882, max timestamp=1732079826893Num uniques 1027; 2024-11-20T05:17:06,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:06,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732079826938, max timestamp=1732079826948Num uniques 1036; 2024-11-20T05:17:06,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-20T05:17:06,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732079826938, max timestamp=1732079826948Num uniques 1036; 2024-11-20T05:17:06,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-20T05:17:06,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732079826938, max timestamp=1732079826948Num uniques 1036; 2024-11-20T05:17:06,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-20T05:17:07,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079826988, max timestamp=1732079827006Num uniques 1045; 2024-11-20T05:17:07,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-20T05:17:07,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079826988, max timestamp=1732079827006Num uniques 1045; 2024-11-20T05:17:07,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-20T05:17:07,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732079826988, max timestamp=1732079827006Num uniques 1045; 2024-11-20T05:17:07,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-20T05:17:07,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732079827044, max timestamp=1732079827070Num uniques 1104; 2024-11-20T05:17:07,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T05:17:07,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732079827044, max timestamp=1732079827070Num uniques 1104; 2024-11-20T05:17:07,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T05:17:07,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732079827044, max timestamp=1732079827070Num uniques 1104; 2024-11-20T05:17:07,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T05:17:07,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732079827133, max timestamp=1732079827150Num uniques 1063; 2024-11-20T05:17:07,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732079827133, max timestamp=1732079827150Num uniques 1063; 2024-11-20T05:17:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:07,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T05:17:07,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T05:17:07,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732079827133, max timestamp=1732079827150Num uniques 1063; 2024-11-20T05:17:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079887151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:07,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T05:17:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:07,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:07,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e0da5f46f61f40488495b236e06b25c2_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e0da5f46f61f40488495b236e06b25c2_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:07,177 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/ed0f46e3c0f545778444119bb5aac520, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:07,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/ed0f46e3c0f545778444119bb5aac520 is 175, key is test_row_0/A:col10/1732079825592/Put/seqid=0 2024-11-20T05:17:07,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742357_1533 (size=39749) 2024-11-20T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732079827236, max timestamp=1732079827258Num uniques 1022; 2024-11-20T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-20T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732079827236, max timestamp=1732079827258Num uniques 1022; 2024-11-20T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-20T05:17:07,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732079827236, max timestamp=1732079827258Num uniques 1022; 2024-11-20T05:17:07,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-20T05:17:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T05:17:07,287 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:07,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:17:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-20T05:17:07,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:17:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T05:17:07,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:17:07,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:17:07,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732079827283, max timestamp=1732079827314Num uniques 1031; 2024-11-20T05:17:07,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-20T05:17:07,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732079827283, max timestamp=1732079827314Num uniques 1031; 2024-11-20T05:17:07,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-20T05:17:07,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732079827283, max timestamp=1732079827314Num uniques 1031; 2024-11-20T05:17:07,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-20T05:17:07,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732079827368, max timestamp=1732079827378Num uniques 1090; 2024-11-20T05:17:07,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-20T05:17:07,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732079827368, max timestamp=1732079827378Num uniques 1090; 2024-11-20T05:17:07,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-20T05:17:07,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732079827368, max timestamp=1732079827378Num uniques 1090; 2024-11-20T05:17:07,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-20T05:17:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T05:17:07,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732079827423, max timestamp=1732079827435Num uniques 1049; 2024-11-20T05:17:07,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732079827423, max timestamp=1732079827435Num uniques 1049; 2024-11-20T05:17:07,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-20T05:17:07,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-20T05:17:07,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732079827423, max timestamp=1732079827435Num uniques 1049; 2024-11-20T05:17:07,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-20T05:17:07,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732079827460, max timestamp=1732079827485Num uniques 1108; 2024-11-20T05:17:07,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732079827460, max timestamp=1732079827485Num uniques 1108; 2024-11-20T05:17:07,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-20T05:17:07,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-20T05:17:07,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732079827460, max timestamp=1732079827485Num uniques 1108; 2024-11-20T05:17:07,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-20T05:17:07,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732079827526, max timestamp=1732079827538Num uniques 1067; 2024-11-20T05:17:07,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-20T05:17:07,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732079827526, max timestamp=1732079827538Num uniques 1067; 2024-11-20T05:17:07,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-20T05:17:07,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732079827526, max timestamp=1732079827538Num uniques 1067; 2024-11-20T05:17:07,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-20T05:17:07,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/ed0f46e3c0f545778444119bb5aac520 2024-11-20T05:17:07,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/560048235c124d49bcb7b4add730e01c is 50, key is test_row_0/B:col10/1732079825592/Put/seqid=0 2024-11-20T05:17:07,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:07,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1732079827577, max timestamp=1732079827591Num uniques 1026; 2024-11-20T05:17:07,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-20T05:17:07,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1732079827577, max timestamp=1732079827591Num uniques 1026; 2024-11-20T05:17:07,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-20T05:17:07,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1732079827577, max timestamp=1732079827591Num uniques 1026; 2024-11-20T05:17:07,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-20T05:17:07,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742358_1534 (size=12151) 2024-11-20T05:17:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T05:17:07,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732079827636, max timestamp=1732079827662Num uniques 1035; 2024-11-20T05:17:07,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-20T05:17:07,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732079827636, max timestamp=1732079827662Num uniques 1035; 2024-11-20T05:17:07,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732079827636, max timestamp=1732079827662Num uniques 1035; 2024-11-20T05:17:07,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-20T05:17:07,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=6870, suffix heap size=135608,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=150 2024-11-20T05:17:07,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732079827698, max timestamp=1732079827718Num uniques 1094; 2024-11-20T05:17:07,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-20T05:17:07,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732079827698, max timestamp=1732079827718Num uniques 1094; 2024-11-20T05:17:07,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-20T05:17:07,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732079827698, max timestamp=1732079827718Num uniques 1009; 2024-11-20T05:17:07,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8884, suffix heap size=132904,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=194 2024-11-20T05:17:07,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:07,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:07,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732079827765, max timestamp=1732079827778Num uniques 1053; 2024-11-20T05:17:07,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-20T05:17:07,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732079827765, max timestamp=1732079827778Num uniques 1053; 2024-11-20T05:17:07,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-20T05:17:07,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732079827765, max timestamp=1732079827778Num uniques 1053; 2024-11-20T05:17:07,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-20T05:17:07,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1732079827827, max timestamp=1732079827848Num uniques 1012; 2024-11-20T05:17:07,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-20T05:17:07,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1732079827827, max timestamp=1732079827848Num uniques 1012; 2024-11-20T05:17:07,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-20T05:17:07,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1732079827827, max timestamp=1732079827848Num uniques 1012; 2024-11-20T05:17:07,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-20T05:17:07,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:07,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T05:17:07,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1732079827895, max timestamp=1732079827921Num uniques 1021; 2024-11-20T05:17:07,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-11-20T05:17:07,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1732079827895, max timestamp=1732079827921Num uniques 1021; 2024-11-20T05:17:07,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1732079827895, max timestamp=1732079827921Num uniques 1021; 2024-11-20T05:17:07,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-11-20T05:17:07,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-11-20T05:17:07,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732079827931, max timestamp=1732079827977Num uniques 1030; 2024-11-20T05:17:07,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=10530, suffix heap size=135088,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=230 2024-11-20T05:17:07,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732079827931, max timestamp=1732079827977Num uniques 1030; 2024-11-20T05:17:07,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=10530, suffix heap size=135088,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=230 2024-11-20T05:17:07,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732079827931, max timestamp=1732079827977Num uniques 1030; 2024-11-20T05:17:07,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=10530, suffix heap size=135088,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=230 2024-11-20T05:17:08,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/560048235c124d49bcb7b4add730e01c 2024-11-20T05:17:08,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/ac7faa734d0e44169c119a040e2374ab is 50, key is test_row_0/C:col10/1732079825592/Put/seqid=0 2024-11-20T05:17:08,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742359_1535 (size=12151) 2024-11-20T05:17:08,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/ac7faa734d0e44169c119a040e2374ab 2024-11-20T05:17:08,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/ed0f46e3c0f545778444119bb5aac520 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/ed0f46e3c0f545778444119bb5aac520 2024-11-20T05:17:08,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/ed0f46e3c0f545778444119bb5aac520, entries=200, sequenceid=250, filesize=38.8 K 2024-11-20T05:17:08,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/560048235c124d49bcb7b4add730e01c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/560048235c124d49bcb7b4add730e01c 2024-11-20T05:17:08,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/560048235c124d49bcb7b4add730e01c, entries=150, sequenceid=250, filesize=11.9 K 2024-11-20T05:17:08,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/ac7faa734d0e44169c119a040e2374ab as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ac7faa734d0e44169c119a040e2374ab 2024-11-20T05:17:08,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ac7faa734d0e44169c119a040e2374ab, entries=150, sequenceid=250, filesize=11.9 K 2024-11-20T05:17:08,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=127.47 KB/130530 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1282ms, sequenceid=250, compaction requested=true 2024-11-20T05:17:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:17:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:17:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:08,031 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T05:17:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:17:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:08,031 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T05:17:08,032 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164307 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T05:17:08,032 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60893 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T05:17:08,032 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A is initiating minor compaction (all files) 2024-11-20T05:17:08,032 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B is initiating minor compaction (all files) 2024-11-20T05:17:08,032 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,032 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,032 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de818a63b80f4eba9faf12d0d184ea49, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/abaebafeacb64315b0c1ffdb122d24bb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/e5e8d5aff3224e20a25fdbff8e1be534, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4754598a790409e8e588f7d0daa183f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/560048235c124d49bcb7b4add730e01c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=59.5 K 2024-11-20T05:17:08,032 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/b5209ec6d86a48bdad2b536da04def9c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a2224a0c332e4a20a826d742caa3d629, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cc0db878a76e4f5dbd6332098e7e97c0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/45f93d78acc84b9390e8e43abb5dd2be, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/ed0f46e3c0f545778444119bb5aac520] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=160.5 K 2024-11-20T05:17:08,032 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,032 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/b5209ec6d86a48bdad2b536da04def9c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a2224a0c332e4a20a826d742caa3d629, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cc0db878a76e4f5dbd6332098e7e97c0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/45f93d78acc84b9390e8e43abb5dd2be, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/ed0f46e3c0f545778444119bb5aac520] 2024-11-20T05:17:08,032 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting de818a63b80f4eba9faf12d0d184ea49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732079820317 2024-11-20T05:17:08,032 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5209ec6d86a48bdad2b536da04def9c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732079820317 2024-11-20T05:17:08,033 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting abaebafeacb64315b0c1ffdb122d24bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732079822087 2024-11-20T05:17:08,033 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2224a0c332e4a20a826d742caa3d629, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732079822087 2024-11-20T05:17:08,033 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e5e8d5aff3224e20a25fdbff8e1be534, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732079822177 2024-11-20T05:17:08,033 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc0db878a76e4f5dbd6332098e7e97c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732079822177 2024-11-20T05:17:08,033 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f4754598a790409e8e588f7d0daa183f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732079823334 2024-11-20T05:17:08,033 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45f93d78acc84b9390e8e43abb5dd2be, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732079823334 2024-11-20T05:17:08,033 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 560048235c124d49bcb7b4add730e01c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732079825592 2024-11-20T05:17:08,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:08,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=147.60 KB heapSize=418.88 KB 2024-11-20T05:17:08,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:08,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T05:17:08,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:08,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T05:17:08,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:08,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T05:17:08,037 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0f46e3c0f545778444119bb5aac520, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732079825592 2024-11-20T05:17:08,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112039948d93088e4c97963551fad0ee5562_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079828026/Put/seqid=0 2024-11-20T05:17:08,047 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#B#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:08,047 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/3f75e489838e4414ab8ca91190192fe1 is 50, key is test_row_0/B:col10/1732079825592/Put/seqid=0 2024-11-20T05:17:08,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,053 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:08,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079888050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079888051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,061 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411204fecc2cc39ef4184a9adc699cc639f92_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:08,063 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411204fecc2cc39ef4184a9adc699cc639f92_ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:08,063 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204fecc2cc39ef4184a9adc699cc639f92_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:08,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742360_1536 (size=14994) 2024-11-20T05:17:08,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:08,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742361_1537 (size=12459) 2024-11-20T05:17:08,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112039948d93088e4c97963551fad0ee5562_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039948d93088e4c97963551fad0ee5562_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:08,070 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/79f73f38809c45cdb984e6dbe2f7919c, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:08,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/79f73f38809c45cdb984e6dbe2f7919c is 175, key is test_row_0/A:col10/1732079828026/Put/seqid=0 2024-11-20T05:17:08,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742362_1538 (size=4469) 2024-11-20T05:17:08,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742363_1539 (size=39949) 2024-11-20T05:17:08,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079888167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079888167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:08,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:08,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:08,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079888377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079888377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T05:17:08,473 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/3f75e489838e4414ab8ca91190192fe1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3f75e489838e4414ab8ca91190192fe1 2024-11-20T05:17:08,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=614, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/79f73f38809c45cdb984e6dbe2f7919c 2024-11-20T05:17:08,485 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#A#compaction#454 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:08,485 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a48f62586bf043e39faf3d730a6cf6cb is 175, key is test_row_0/A:col10/1732079825592/Put/seqid=0 2024-11-20T05:17:08,486 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 3f75e489838e4414ab8ca91190192fe1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:08,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:08,486 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, priority=11, startTime=1732079828031; duration=0sec 2024-11-20T05:17:08,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:08,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B 2024-11-20T05:17:08,487 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T05:17:08,492 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60893 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T05:17:08,492 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C is initiating minor compaction (all files) 2024-11-20T05:17:08,492 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,492 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0a0ddb32b5e14e5eb16e9239de0c7df5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/6c14e24161d54a0db44587610cd95f0c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e9d228c62cd247b995037ff699cef550, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d0766cbbe2eb4c0e80a8934b2f9e4367, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ac7faa734d0e44169c119a040e2374ab] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=59.5 K 2024-11-20T05:17:08,493 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a0ddb32b5e14e5eb16e9239de0c7df5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732079820317 2024-11-20T05:17:08,493 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c14e24161d54a0db44587610cd95f0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732079822087 2024-11-20T05:17:08,493 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d228c62cd247b995037ff699cef550, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732079822177 2024-11-20T05:17:08,493 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d0766cbbe2eb4c0e80a8934b2f9e4367, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732079823334 2024-11-20T05:17:08,494 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ac7faa734d0e44169c119a040e2374ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732079825592 2024-11-20T05:17:08,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742364_1540 (size=31413) 2024-11-20T05:17:08,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/a3ef6abe92bb44a6947e96f1f552a6ae is 50, key is test_row_0/B:col10/1732079828026/Put/seqid=0 2024-11-20T05:17:08,507 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a48f62586bf043e39faf3d730a6cf6cb as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a48f62586bf043e39faf3d730a6cf6cb 2024-11-20T05:17:08,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,515 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#C#compaction#456 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:08,515 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/f02317817e224e939975060bac9e21d8 is 50, key is test_row_0/C:col10/1732079825592/Put/seqid=0 2024-11-20T05:17:08,518 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into a48f62586bf043e39faf3d730a6cf6cb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:08,518 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:08,518 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, priority=11, startTime=1732079828031; duration=0sec 2024-11-20T05:17:08,518 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:08,518 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A 2024-11-20T05:17:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079888527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,529 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_0' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079888527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,534 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079888537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,542 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742365_1541 (size=12301) 2024-11-20T05:17:08,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/a3ef6abe92bb44a6947e96f1f552a6ae 2024-11-20T05:17:08,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742366_1542 (size=12459) 2024-11-20T05:17:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/fdbebcd60e1c4afbb2721f6c32e79922 is 50, key is test_row_0/C:col10/1732079828026/Put/seqid=0 2024-11-20T05:17:08,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742367_1543 (size=12301) 2024-11-20T05:17:08,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:08,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:08,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079888688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079888688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:08,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,962 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/f02317817e224e939975060bac9e21d8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/f02317817e224e939975060bac9e21d8 2024-11-20T05:17:08,968 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into f02317817e224e939975060bac9e21d8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:08,968 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:08,968 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, priority=11, startTime=1732079828031; duration=0sec 2024-11-20T05:17:08,968 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:08,968 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C 2024-11-20T05:17:08,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:08,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:08,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:08,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:08,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/fdbebcd60e1c4afbb2721f6c32e79922 2024-11-20T05:17:08,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/79f73f38809c45cdb984e6dbe2f7919c as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/79f73f38809c45cdb984e6dbe2f7919c 2024-11-20T05:17:08,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/79f73f38809c45cdb984e6dbe2f7919c, entries=200, sequenceid=614, filesize=39.0 K 2024-11-20T05:17:08,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/a3ef6abe92bb44a6947e96f1f552a6ae as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a3ef6abe92bb44a6947e96f1f552a6ae 2024-11-20T05:17:08,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a3ef6abe92bb44a6947e96f1f552a6ae, entries=150, sequenceid=614, filesize=12.0 K 2024-11-20T05:17:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/fdbebcd60e1c4afbb2721f6c32e79922 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/fdbebcd60e1c4afbb2721f6c32e79922 2024-11-20T05:17:09,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/fdbebcd60e1c4afbb2721f6c32e79922, entries=150, sequenceid=614, filesize=12.0 K 2024-11-20T05:17:09,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~418.83 KB/428880, currentSize=33.54 KB/34350 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 975ms, sequenceid=614, compaction requested=false 2024-11-20T05:17:09,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:09,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T05:17:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:09,129 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T05:17:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112068c49d24a52a47a6a081bb783034e87a_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079828048/Put/seqid=0 2024-11-20T05:17:09,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742368_1544 (size=12454) 2024-11-20T05:17:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:09,206 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112068c49d24a52a47a6a081bb783034e87a_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112068c49d24a52a47a6a081bb783034e87a_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:09,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/213f567131e74d048368b4d06d0313a4, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:09,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/213f567131e74d048368b4d06d0313a4 is 175, key is test_row_0/A:col10/1732079828048/Put/seqid=0 2024-11-20T05:17:09,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742369_1545 (size=31255) 2024-11-20T05:17:09,240 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=625, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/213f567131e74d048368b4d06d0313a4 2024-11-20T05:17:09,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:09,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/f4d11e5bd9a9465da7ac4b86604036fa is 50, key is test_row_0/B:col10/1732079828048/Put/seqid=0 2024-11-20T05:17:09,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742370_1546 (size=12301) 2024-11-20T05:17:09,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079829298, max timestamp=1732079829311Num uniques 859; 2024-11-20T05:17:09,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:09,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732079829298, max timestamp=1732079829311Num uniques 859; 2024-11-20T05:17:09,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T05:17:09,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079889363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:09,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079889364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T05:17:09,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079889477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:09,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079889477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:09,672 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/f4d11e5bd9a9465da7ac4b86604036fa 2024-11-20T05:17:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a293929d6b1444ad9d66e393a85388f7 is 50, key is test_row_0/C:col10/1732079828048/Put/seqid=0 2024-11-20T05:17:09,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079889687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 341 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079889687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:09,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742371_1547 (size=12301) 2024-11-20T05:17:10,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079889997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079889997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,105 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a293929d6b1444ad9d66e393a85388f7 2024-11-20T05:17:10,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/213f567131e74d048368b4d06d0313a4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/213f567131e74d048368b4d06d0313a4 2024-11-20T05:17:10,110 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/213f567131e74d048368b4d06d0313a4, entries=150, sequenceid=625, filesize=30.5 K 2024-11-20T05:17:10,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/f4d11e5bd9a9465da7ac4b86604036fa as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4d11e5bd9a9465da7ac4b86604036fa 2024-11-20T05:17:10,113 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4d11e5bd9a9465da7ac4b86604036fa, entries=150, sequenceid=625, filesize=12.0 K 2024-11-20T05:17:10,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a293929d6b1444ad9d66e393a85388f7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a293929d6b1444ad9d66e393a85388f7 2024-11-20T05:17:10,117 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a293929d6b1444ad9d66e393a85388f7, entries=150, sequenceid=625, filesize=12.0 K 2024-11-20T05:17:10,117 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=145.36 KB/148850 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 988ms, sequenceid=625, compaction requested=true 2024-11-20T05:17:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-20T05:17:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-20T05:17:10,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-20T05:17:10,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8290 sec 2024-11-20T05:17:10,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.8330 sec 2024-11-20T05:17:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:10,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=152.07 KB heapSize=443.08 KB 2024-11-20T05:17:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:10,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112063c78b4a2d5a4c8faee74ffba84c2c56_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079830510/Put/seqid=0 2024-11-20T05:17:10,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742372_1548 (size=14994) 2024-11-20T05:17:10,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079890526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079890533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079890637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079890649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079890847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079890858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:10,925 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:10,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112063c78b4a2d5a4c8faee74ffba84c2c56_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112063c78b4a2d5a4c8faee74ffba84c2c56_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:10,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/563cdb729e5b46dcb28127443eac6d0f, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:10,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/563cdb729e5b46dcb28127443eac6d0f is 175, key is test_row_0/A:col10/1732079830510/Put/seqid=0 2024-11-20T05:17:10,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742373_1549 (size=39949) 2024-11-20T05:17:10,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=660, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/563cdb729e5b46dcb28127443eac6d0f 2024-11-20T05:17:10,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/1cab20ecdabe4f569f8af4284ef3fbb5 is 50, key is test_row_0/B:col10/1732079830510/Put/seqid=0 2024-11-20T05:17:10,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742374_1550 (size=12301) 2024-11-20T05:17:11,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079891157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:11,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079891167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:11,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/1cab20ecdabe4f569f8af4284ef3fbb5 2024-11-20T05:17:11,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/0cc54b785aa944a896a5ae9aa30e19f4 is 50, key is test_row_0/C:col10/1732079830510/Put/seqid=0 2024-11-20T05:17:11,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742375_1551 (size=12301) 2024-11-20T05:17:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T05:17:11,458 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:11,458 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:17:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-20T05:17:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T05:17:11,465 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:17:11,466 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:17:11,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:17:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T05:17:11,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-20T05:17:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 348 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079891667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079891677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:11,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.56 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/0cc54b785aa944a896a5ae9aa30e19f4 2024-11-20T05:17:11,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/563cdb729e5b46dcb28127443eac6d0f as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/563cdb729e5b46dcb28127443eac6d0f 2024-11-20T05:17:11,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/563cdb729e5b46dcb28127443eac6d0f, entries=200, sequenceid=660, filesize=39.0 K 2024-11-20T05:17:11,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/1cab20ecdabe4f569f8af4284ef3fbb5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1cab20ecdabe4f569f8af4284ef3fbb5 2024-11-20T05:17:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1cab20ecdabe4f569f8af4284ef3fbb5, entries=150, sequenceid=660, filesize=12.0 K 2024-11-20T05:17:11,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/0cc54b785aa944a896a5ae9aa30e19f4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0cc54b785aa944a896a5ae9aa30e19f4 2024-11-20T05:17:11,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-20T05:17:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0cc54b785aa944a896a5ae9aa30e19f4, entries=150, sequenceid=660, filesize=12.0 K 2024-11-20T05:17:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~443.03 KB/453664, currentSize=26.84 KB/27480 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1262ms, sequenceid=660, compaction requested=true 2024-11-20T05:17:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:11,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:17:11,773 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:17:11,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:11,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:17:11,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:11,773 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:17:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:17:11,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,775 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:17:11,775 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:17:11,775 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A is initiating minor compaction (all files) 2024-11-20T05:17:11,775 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B is initiating minor compaction (all files) 2024-11-20T05:17:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,775 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,775 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,775 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a48f62586bf043e39faf3d730a6cf6cb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/79f73f38809c45cdb984e6dbe2f7919c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/213f567131e74d048368b4d06d0313a4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/563cdb729e5b46dcb28127443eac6d0f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=139.2 K 2024-11-20T05:17:11,775 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3f75e489838e4414ab8ca91190192fe1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a3ef6abe92bb44a6947e96f1f552a6ae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4d11e5bd9a9465da7ac4b86604036fa, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1cab20ecdabe4f569f8af4284ef3fbb5] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=48.2 K 2024-11-20T05:17:11,775 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,775 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a48f62586bf043e39faf3d730a6cf6cb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/79f73f38809c45cdb984e6dbe2f7919c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/213f567131e74d048368b4d06d0313a4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/563cdb729e5b46dcb28127443eac6d0f] 2024-11-20T05:17:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,775 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f75e489838e4414ab8ca91190192fe1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732079825592 2024-11-20T05:17:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,776 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48f62586bf043e39faf3d730a6cf6cb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732079825592 2024-11-20T05:17:11,776 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a3ef6abe92bb44a6947e96f1f552a6ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732079828026 2024-11-20T05:17:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,776 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f73f38809c45cdb984e6dbe2f7919c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732079828011 2024-11-20T05:17:11,776 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d11e5bd9a9465da7ac4b86604036fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1732079828043 2024-11-20T05:17:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,776 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 213f567131e74d048368b4d06d0313a4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1732079828043 2024-11-20T05:17:11,776 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cab20ecdabe4f569f8af4284ef3fbb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1732079829361 2024-11-20T05:17:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,776 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563cdb729e5b46dcb28127443eac6d0f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1732079829355 2024-11-20T05:17:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T05:17:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,782 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,783 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112014b0656377c5421f8a1153ba841a7f5e_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:11,783 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,784 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/01428b2cda2342009b91e0d297c946c2 is 50, key is test_row_0/B:col10/1732079830510/Put/seqid=0 2024-11-20T05:17:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,786 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112014b0656377c5421f8a1153ba841a7f5e_ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,786 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112014b0656377c5421f8a1153ba841a7f5e_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742377_1553 (size=4469) 2024-11-20T05:17:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,790 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#A#compaction#464 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,790 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/220b291ccdea4534b58dac07e935639b is 175, key is test_row_0/A:col10/1732079830510/Put/seqid=0 2024-11-20T05:17:11,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742376_1552 (size=12745) 2024-11-20T05:17:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,798 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/01428b2cda2342009b91e0d297c946c2 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/01428b2cda2342009b91e0d297c946c2 2024-11-20T05:17:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,804 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 01428b2cda2342009b91e0d297c946c2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:11,804 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:11,804 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, priority=12, startTime=1732079831773; duration=0sec 2024-11-20T05:17:11,804 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:11,804 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B 2024-11-20T05:17:11,805 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T05:17:11,806 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T05:17:11,806 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C is initiating minor compaction (all files) 2024-11-20T05:17:11,806 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,806 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/f02317817e224e939975060bac9e21d8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/fdbebcd60e1c4afbb2721f6c32e79922, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a293929d6b1444ad9d66e393a85388f7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0cc54b785aa944a896a5ae9aa30e19f4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=48.2 K 2024-11-20T05:17:11,806 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting f02317817e224e939975060bac9e21d8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732079825592 2024-11-20T05:17:11,807 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting fdbebcd60e1c4afbb2721f6c32e79922, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732079828026 2024-11-20T05:17:11,807 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a293929d6b1444ad9d66e393a85388f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1732079828043 2024-11-20T05:17:11,807 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc54b785aa944a896a5ae9aa30e19f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1732079829361 2024-11-20T05:17:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,839 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:11,839 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a803fba86cc1444daf00a66af82b6f7a is 50, key is test_row_0/C:col10/1732079830510/Put/seqid=0 2024-11-20T05:17:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742378_1554 (size=31699) 2024-11-20T05:17:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742379_1555 (size=12745) 2024-11-20T05:17:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,867 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a803fba86cc1444daf00a66af82b6f7a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a803fba86cc1444daf00a66af82b6f7a 2024-11-20T05:17:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,873 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into a803fba86cc1444daf00a66af82b6f7a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:11,873 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:11,873 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, priority=12, startTime=1732079831773; duration=0sec 2024-11-20T05:17:11,873 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:11,873 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C 2024-11-20T05:17:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-20T05:17:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:11,924 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T05:17:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de4afcb1b4734d49a1a0b85e2e8981c9_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079830524/Put/seqid=0 2024-11-20T05:17:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742380_1556 (size=9914) 2024-11-20T05:17:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T05:17:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,247 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/220b291ccdea4534b58dac07e935639b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/220b291ccdea4534b58dac07e935639b 2024-11-20T05:17:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,253 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 220b291ccdea4534b58dac07e935639b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:12,253 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:12,253 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, priority=12, startTime=1732079831773; duration=0sec 2024-11-20T05:17:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,253 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:12,253 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A 2024-11-20T05:17:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,344 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de4afcb1b4734d49a1a0b85e2e8981c9_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de4afcb1b4734d49a1a0b85e2e8981c9_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/895f0dbdf17b4e4f895fc60cdbc97e89, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:12,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/895f0dbdf17b4e4f895fc60cdbc97e89 is 175, key is test_row_0/A:col10/1732079830524/Put/seqid=0 2024-11-20T05:17:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742381_1557 (size=22561) 2024-11-20T05:17:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T05:17:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:12,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079832707, max timestamp=1732079832726Num uniques 859; 2024-11-20T05:17:12,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:17:12,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079832707, max timestamp=1732079832726Num uniques 859; 2024-11-20T05:17:12,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:17:12,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732079832707, max timestamp=1732079832726Num uniques 859; 2024-11-20T05:17:12,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T05:17:12,771 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=669, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/895f0dbdf17b4e4f895fc60cdbc97e89 2024-11-20T05:17:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079832749, max timestamp=1732079832770Num uniques 1068; 2024-11-20T05:17:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T05:17:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079832749, max timestamp=1732079832770Num uniques 1068; 2024-11-20T05:17:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T05:17:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732079832749, max timestamp=1732079832770Num uniques 1068; 2024-11-20T05:17:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T05:17:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/8aa30da516b8440ca1e5fec21862e970 is 50, key is test_row_0/B:col10/1732079830524/Put/seqid=0 2024-11-20T05:17:12,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742382_1558 (size=9857) 2024-11-20T05:17:12,786 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/8aa30da516b8440ca1e5fec21862e970 2024-11-20T05:17:12,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/d79ff5f99e5843579876de6e444410b4 is 50, key is test_row_0/C:col10/1732079830524/Put/seqid=0 2024-11-20T05:17:12,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742383_1559 (size=9857) 2024-11-20T05:17:12,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079832807, max timestamp=1732079832838Num uniques 1027; 2024-11-20T05:17:12,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:12,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079832807, max timestamp=1732079832838Num uniques 1027; 2024-11-20T05:17:12,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732079832807, max timestamp=1732079832838Num uniques 1027; 2024-11-20T05:17:12,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:12,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T05:17:12,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732079832869, max timestamp=1732079832899Num uniques 1036; 2024-11-20T05:17:12,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732079832869, max timestamp=1732079832899Num uniques 1036; 2024-11-20T05:17:12,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T05:17:12,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T05:17:12,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732079832869, max timestamp=1732079832899Num uniques 1036; 2024-11-20T05:17:12,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T05:17:12,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732079832938, max timestamp=1732079832956Num uniques 1095; 2024-11-20T05:17:12,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-20T05:17:12,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732079832938, max timestamp=1732079832956Num uniques 1095; 2024-11-20T05:17:12,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-20T05:17:12,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8923, totalHeapSize=20520, min timestamp=1732079832938, max timestamp=1732079832956Num uniques 1095; 2024-11-20T05:17:12,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50143, new segment data size=8923, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-20T05:17:13,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732079832975, max timestamp=1732079833005Num uniques 1054; 2024-11-20T05:17:13,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-20T05:17:13,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732079832975, max timestamp=1732079833005Num uniques 1054; 2024-11-20T05:17:13,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-20T05:17:13,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732079832975, max timestamp=1732079833005Num uniques 1054; 2024-11-20T05:17:13,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-20T05:17:13,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732079833037, max timestamp=1732079833059Num uniques 1063; 2024-11-20T05:17:13,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T05:17:13,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732079833037, max timestamp=1732079833059Num uniques 1063; 2024-11-20T05:17:13,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T05:17:13,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732079833037, max timestamp=1732079833059Num uniques 1063; 2024-11-20T05:17:13,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T05:17:13,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732079833073, max timestamp=1732079833108Num uniques 1022; 2024-11-20T05:17:13,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-20T05:17:13,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732079833073, max timestamp=1732079833108Num uniques 1022; 2024-11-20T05:17:13,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732079833073, max timestamp=1732079833108Num uniques 1022; 2024-11-20T05:17:13,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-20T05:17:13,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-20T05:17:13,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732079833162, max timestamp=1732079833174Num uniques 1081; 2024-11-20T05:17:13,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-20T05:17:13,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732079833162, max timestamp=1732079833174Num uniques 1081; 2024-11-20T05:17:13,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-20T05:17:13,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732079833162, max timestamp=1732079833174Num uniques 1081; 2024-11-20T05:17:13,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51d5d2011a1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-20T05:17:13,199 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/d79ff5f99e5843579876de6e444410b4 2024-11-20T05:17:13,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/895f0dbdf17b4e4f895fc60cdbc97e89 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/895f0dbdf17b4e4f895fc60cdbc97e89 2024-11-20T05:17:13,208 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/895f0dbdf17b4e4f895fc60cdbc97e89, entries=100, sequenceid=669, filesize=22.0 K 2024-11-20T05:17:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/8aa30da516b8440ca1e5fec21862e970 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8aa30da516b8440ca1e5fec21862e970 2024-11-20T05:17:13,213 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8aa30da516b8440ca1e5fec21862e970, entries=100, sequenceid=669, filesize=9.6 K 2024-11-20T05:17:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/d79ff5f99e5843579876de6e444410b4 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d79ff5f99e5843579876de6e444410b4 2024-11-20T05:17:13,217 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d79ff5f99e5843579876de6e444410b4, entries=100, sequenceid=669, filesize=9.6 K 2024-11-20T05:17:13,223 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=67.09 KB/68700 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1298ms, sequenceid=669, compaction requested=false 2024-11-20T05:17:13,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:13,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:13,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-20T05:17:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-20T05:17:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:13,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=80.51 KB heapSize=223.55 KB 2024-11-20T05:17:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T05:17:13,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-20T05:17:13,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7590 sec 2024-11-20T05:17:13,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.7710 sec 2024-11-20T05:17:13,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203855960d0a284fe69fad72dfe4bcb8da_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079833185/Put/seqid=0 2024-11-20T05:17:13,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742384_1560 (size=17534) 2024-11-20T05:17:13,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:13,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203855960d0a284fe69fad72dfe4bcb8da_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203855960d0a284fe69fad72dfe4bcb8da_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:13,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/6248505bd44e4240a92b81bebb52e57e, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:13,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/6248505bd44e4240a92b81bebb52e57e is 175, key is test_row_0/A:col10/1732079833185/Put/seqid=0 2024-11-20T05:17:13,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742385_1561 (size=48639) 2024-11-20T05:17:13,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079893277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 438 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079893279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 383 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079893387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 439 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079893387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 440 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079893597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T05:17:13,607 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:13,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:17:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-20T05:17:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T05:17:13,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:17:13,612 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:17:13,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:17:13,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 384 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079893607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=836, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/6248505bd44e4240a92b81bebb52e57e 2024-11-20T05:17:13,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/5b2309d677bd428395a36bfcc16375e3 is 50, key is test_row_0/B:col10/1732079833185/Put/seqid=0 2024-11-20T05:17:13,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742386_1562 (size=12301) 2024-11-20T05:17:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T05:17:13,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T05:17:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:13,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:13,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:13,791 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T05:17:13,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 441 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079893907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T05:17:13,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:13,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:13,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079893917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T05:17:14,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T05:17:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:14,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:14,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/5b2309d677bd428395a36bfcc16375e3 2024-11-20T05:17:14,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/ba27157a090642a593a169aed7b129de is 50, key is test_row_0/C:col10/1732079833185/Put/seqid=0 2024-11-20T05:17:14,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742387_1563 (size=12301) 2024-11-20T05:17:14,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/ba27157a090642a593a169aed7b129de 2024-11-20T05:17:14,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/6248505bd44e4240a92b81bebb52e57e as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/6248505bd44e4240a92b81bebb52e57e 2024-11-20T05:17:14,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/6248505bd44e4240a92b81bebb52e57e, entries=250, sequenceid=836, filesize=47.5 K 2024-11-20T05:17:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/5b2309d677bd428395a36bfcc16375e3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/5b2309d677bd428395a36bfcc16375e3 2024-11-20T05:17:14,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/5b2309d677bd428395a36bfcc16375e3, entries=150, sequenceid=836, filesize=12.0 K 2024-11-20T05:17:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/ba27157a090642a593a169aed7b129de as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ba27157a090642a593a169aed7b129de 2024-11-20T05:17:14,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ba27157a090642a593a169aed7b129de, entries=150, sequenceid=836, filesize=12.0 K 2024-11-20T05:17:14,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~223.50 KB/228864, currentSize=100.63 KB/103050 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 909ms, sequenceid=836, compaction requested=true 2024-11-20T05:17:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:17:14,132 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:17:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:14,133 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:14,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:17:14,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:14,133 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:14,133 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:14,133 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A is initiating minor compaction (all files) 2024-11-20T05:17:14,133 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B is initiating minor compaction (all files) 2024-11-20T05:17:14,133 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:14,133 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:14,133 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/220b291ccdea4534b58dac07e935639b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/895f0dbdf17b4e4f895fc60cdbc97e89, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/6248505bd44e4240a92b81bebb52e57e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=100.5 K 2024-11-20T05:17:14,133 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/01428b2cda2342009b91e0d297c946c2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8aa30da516b8440ca1e5fec21862e970, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/5b2309d677bd428395a36bfcc16375e3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=34.1 K 2024-11-20T05:17:14,133 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:14,133 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/220b291ccdea4534b58dac07e935639b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/895f0dbdf17b4e4f895fc60cdbc97e89, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/6248505bd44e4240a92b81bebb52e57e] 2024-11-20T05:17:14,134 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 01428b2cda2342009b91e0d297c946c2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1732079829361 2024-11-20T05:17:14,134 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 220b291ccdea4534b58dac07e935639b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1732079829361 2024-11-20T05:17:14,134 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa30da516b8440ca1e5fec21862e970, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732079830524 2024-11-20T05:17:14,135 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 895f0dbdf17b4e4f895fc60cdbc97e89, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732079830524 2024-11-20T05:17:14,135 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b2309d677bd428395a36bfcc16375e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1732079833185 2024-11-20T05:17:14,135 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6248505bd44e4240a92b81bebb52e57e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1732079833183 2024-11-20T05:17:14,139 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:14,140 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120cd777c06b828425595440dba32e86cc8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:14,141 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:14,142 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/aad7b1be6e5d4bfd904d8aba1ca4606d is 50, key is test_row_0/B:col10/1732079833185/Put/seqid=0 2024-11-20T05:17:14,142 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120cd777c06b828425595440dba32e86cc8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:14,142 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cd777c06b828425595440dba32e86cc8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:14,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742389_1565 (size=4469) 2024-11-20T05:17:14,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742388_1564 (size=12847) 2024-11-20T05:17:14,146 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#A#compaction#473 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:14,146 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/3ae591557f1a4f86afb1e4995fa46a07 is 175, key is test_row_0/A:col10/1732079833185/Put/seqid=0 2024-11-20T05:17:14,152 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/aad7b1be6e5d4bfd904d8aba1ca4606d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/aad7b1be6e5d4bfd904d8aba1ca4606d 2024-11-20T05:17:14,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742390_1566 (size=31801) 2024-11-20T05:17:14,156 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into aad7b1be6e5d4bfd904d8aba1ca4606d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:14,156 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:14,156 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, priority=13, startTime=1732079834132; duration=0sec 2024-11-20T05:17:14,157 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:14,157 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B 2024-11-20T05:17:14,157 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:14,158 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:14,158 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C is initiating minor compaction (all files) 2024-11-20T05:17:14,158 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:14,158 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a803fba86cc1444daf00a66af82b6f7a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d79ff5f99e5843579876de6e444410b4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ba27157a090642a593a169aed7b129de] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=34.1 K 2024-11-20T05:17:14,159 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a803fba86cc1444daf00a66af82b6f7a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1732079829361 2024-11-20T05:17:14,159 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting d79ff5f99e5843579876de6e444410b4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732079830524 2024-11-20T05:17:14,160 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting ba27157a090642a593a169aed7b129de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1732079833185 2024-11-20T05:17:14,175 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:14,176 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/e0dc2815a8774955bd12a732db1b7c14 is 50, key is test_row_0/C:col10/1732079833185/Put/seqid=0 2024-11-20T05:17:14,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742391_1567 (size=12847) 2024-11-20T05:17:14,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T05:17:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:14,225 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T05:17:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:14,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c4433cdbc6ea4fffa6137542cac36670_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079833276/Put/seqid=0 2024-11-20T05:17:14,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742392_1568 (size=12454) 2024-11-20T05:17:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T05:17:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:14,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 390 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079894446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:14,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 449 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079894450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:14,557 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/3ae591557f1a4f86afb1e4995fa46a07 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3ae591557f1a4f86afb1e4995fa46a07 2024-11-20T05:17:14,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 391 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079894557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:14,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 450 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079894558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:14,561 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 3ae591557f1a4f86afb1e4995fa46a07(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:14,561 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:14,561 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, priority=13, startTime=1732079834132; duration=0sec 2024-11-20T05:17:14,561 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:14,561 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A 2024-11-20T05:17:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:14,583 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/e0dc2815a8774955bd12a732db1b7c14 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e0dc2815a8774955bd12a732db1b7c14 2024-11-20T05:17:14,588 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into e0dc2815a8774955bd12a732db1b7c14(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:14,588 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:14,588 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, priority=13, startTime=1732079834132; duration=0sec 2024-11-20T05:17:14,589 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:14,589 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C 2024-11-20T05:17:14,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:14,638 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c4433cdbc6ea4fffa6137542cac36670_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c4433cdbc6ea4fffa6137542cac36670_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:14,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cb1530c9621f441e96572f4b45c2aebe, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:14,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cb1530c9621f441e96572f4b45c2aebe is 175, key is test_row_0/A:col10/1732079833276/Put/seqid=0 2024-11-20T05:17:14,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742393_1569 (size=31255) 2024-11-20T05:17:14,666 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=855, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cb1530c9621f441e96572f4b45c2aebe 2024-11-20T05:17:14,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/6d97fee72cad4502884ec2cdcd19a2f5 is 50, key is test_row_0/B:col10/1732079833276/Put/seqid=0 2024-11-20T05:17:14,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742394_1570 (size=12301) 2024-11-20T05:17:14,716 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=855 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/6d97fee72cad4502884ec2cdcd19a2f5 2024-11-20T05:17:14,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/2d23047d1a1b4e4798fbee9b63ceef89 is 50, key is test_row_0/C:col10/1732079833276/Put/seqid=0 2024-11-20T05:17:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T05:17:14,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742395_1571 (size=12301) 2024-11-20T05:17:14,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 392 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079894767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 451 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079894772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 393 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079895077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 452 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079895087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,166 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=855 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/2d23047d1a1b4e4798fbee9b63ceef89 2024-11-20T05:17:15,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/cb1530c9621f441e96572f4b45c2aebe as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cb1530c9621f441e96572f4b45c2aebe 2024-11-20T05:17:15,171 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cb1530c9621f441e96572f4b45c2aebe, entries=150, sequenceid=855, filesize=30.5 K 2024-11-20T05:17:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/6d97fee72cad4502884ec2cdcd19a2f5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/6d97fee72cad4502884ec2cdcd19a2f5 2024-11-20T05:17:15,174 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/6d97fee72cad4502884ec2cdcd19a2f5, entries=150, sequenceid=855, filesize=12.0 K 2024-11-20T05:17:15,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/2d23047d1a1b4e4798fbee9b63ceef89 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/2d23047d1a1b4e4798fbee9b63ceef89 2024-11-20T05:17:15,178 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/2d23047d1a1b4e4798fbee9b63ceef89, entries=150, sequenceid=855, filesize=12.0 K 2024-11-20T05:17:15,178 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 953ms, sequenceid=855, compaction requested=false 2024-11-20T05:17:15,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:15,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:15,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-20T05:17:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-20T05:17:15,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-20T05:17:15,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5680 sec 2024-11-20T05:17:15,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.5740 sec 2024-11-20T05:17:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:15,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T05:17:15,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:15,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:15,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:15,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:15,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:15,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:15,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d34229d3f40e4d55b9fefc796ff8f581_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079834445/Put/seqid=0 2024-11-20T05:17:15,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742396_1572 (size=12454) 2024-11-20T05:17:15,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 403 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079895652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 459 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079895653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T05:17:15,758 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:15,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:17:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-20T05:17:15,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:17:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T05:17:15,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:17:15,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:17:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 404 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079895767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 460 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079895767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T05:17:15,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T05:17:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:15,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:15,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 405 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079895977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 461 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079895977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:16,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:16,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d34229d3f40e4d55b9fefc796ff8f581_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d34229d3f40e4d55b9fefc796ff8f581_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:16,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/24075bf038a74ccebc390ef72795ce38, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:16,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/24075bf038a74ccebc390ef72795ce38 is 175, key is test_row_0/A:col10/1732079834445/Put/seqid=0 2024-11-20T05:17:16,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742397_1573 (size=31255) 2024-11-20T05:17:16,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=872, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/24075bf038a74ccebc390ef72795ce38 2024-11-20T05:17:16,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/b14869023ac3440cab8276c4be2c62d3 is 50, key is test_row_0/B:col10/1732079834445/Put/seqid=0 2024-11-20T05:17:16,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742398_1574 (size=12301) 2024-11-20T05:17:16,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T05:17:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T05:17:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T05:17:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 406 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079896287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 462 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079896287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:16,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T05:17:16,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:16,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T05:17:16,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/b14869023ac3440cab8276c4be2c62d3 2024-11-20T05:17:16,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8e627bac46a748d698f2332fecba18de is 50, key is test_row_0/C:col10/1732079834445/Put/seqid=0 2024-11-20T05:17:16,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742399_1575 (size=12301) 2024-11-20T05:17:16,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8e627bac46a748d698f2332fecba18de 2024-11-20T05:17:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/24075bf038a74ccebc390ef72795ce38 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/24075bf038a74ccebc390ef72795ce38 2024-11-20T05:17:16,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/24075bf038a74ccebc390ef72795ce38, entries=150, sequenceid=872, filesize=30.5 K 2024-11-20T05:17:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/b14869023ac3440cab8276c4be2c62d3 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/b14869023ac3440cab8276c4be2c62d3 2024-11-20T05:17:16,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/b14869023ac3440cab8276c4be2c62d3, entries=150, sequenceid=872, filesize=12.0 K 2024-11-20T05:17:16,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8e627bac46a748d698f2332fecba18de as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8e627bac46a748d698f2332fecba18de 2024-11-20T05:17:16,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8e627bac46a748d698f2332fecba18de, entries=150, sequenceid=872, filesize=12.0 K 2024-11-20T05:17:16,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 896ms, sequenceid=872, compaction requested=true 2024-11-20T05:17:16,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:17:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:17:16,486 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:16,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:17:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:16,486 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:16,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:16,486 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A is initiating minor compaction (all files) 2024-11-20T05:17:16,486 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B is initiating minor compaction (all files) 2024-11-20T05:17:16,486 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,486 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,486 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3ae591557f1a4f86afb1e4995fa46a07, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cb1530c9621f441e96572f4b45c2aebe, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/24075bf038a74ccebc390ef72795ce38] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=92.1 K 2024-11-20T05:17:16,486 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/aad7b1be6e5d4bfd904d8aba1ca4606d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/6d97fee72cad4502884ec2cdcd19a2f5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/b14869023ac3440cab8276c4be2c62d3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=36.6 K 2024-11-20T05:17:16,486 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,486 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3ae591557f1a4f86afb1e4995fa46a07, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cb1530c9621f441e96572f4b45c2aebe, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/24075bf038a74ccebc390ef72795ce38] 2024-11-20T05:17:16,487 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting aad7b1be6e5d4bfd904d8aba1ca4606d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1732079833185 2024-11-20T05:17:16,487 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae591557f1a4f86afb1e4995fa46a07, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1732079833185 2024-11-20T05:17:16,487 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d97fee72cad4502884ec2cdcd19a2f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=855, earliestPutTs=1732079833269 2024-11-20T05:17:16,487 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1530c9621f441e96572f4b45c2aebe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=855, earliestPutTs=1732079833269 2024-11-20T05:17:16,487 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting b14869023ac3440cab8276c4be2c62d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1732079834445 2024-11-20T05:17:16,487 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24075bf038a74ccebc390ef72795ce38, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1732079834445 2024-11-20T05:17:16,491 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:16,493 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:16,493 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120df568019bee14c7083690d764baf4c96_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:16,493 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/bb83438266404caab2314d174b4be764 is 50, key is test_row_0/B:col10/1732079834445/Put/seqid=0 2024-11-20T05:17:16,495 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120df568019bee14c7083690d764baf4c96_ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:16,495 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120df568019bee14c7083690d764baf4c96_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:16,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742400_1576 (size=12949) 2024-11-20T05:17:16,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742401_1577 (size=4469) 2024-11-20T05:17:16,498 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#A#compaction#483 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:16,499 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/09821cdd622a423e92d02bf1d1381289 is 175, key is test_row_0/A:col10/1732079834445/Put/seqid=0 2024-11-20T05:17:16,500 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/bb83438266404caab2314d174b4be764 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/bb83438266404caab2314d174b4be764 2024-11-20T05:17:16,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742402_1578 (size=31903) 2024-11-20T05:17:16,512 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into bb83438266404caab2314d174b4be764(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:16,512 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:16,512 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, priority=13, startTime=1732079836486; duration=0sec 2024-11-20T05:17:16,512 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:16,512 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B 2024-11-20T05:17:16,512 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:16,514 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/09821cdd622a423e92d02bf1d1381289 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/09821cdd622a423e92d02bf1d1381289 2024-11-20T05:17:16,517 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:16,517 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C is initiating minor compaction (all files) 2024-11-20T05:17:16,517 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,517 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e0dc2815a8774955bd12a732db1b7c14, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/2d23047d1a1b4e4798fbee9b63ceef89, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8e627bac46a748d698f2332fecba18de] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=36.6 K 2024-11-20T05:17:16,517 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting e0dc2815a8774955bd12a732db1b7c14, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1732079833185 2024-11-20T05:17:16,518 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d23047d1a1b4e4798fbee9b63ceef89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=855, earliestPutTs=1732079833269 2024-11-20T05:17:16,518 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e627bac46a748d698f2332fecba18de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1732079834445 2024-11-20T05:17:16,519 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 09821cdd622a423e92d02bf1d1381289(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:16,520 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:16,520 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, priority=13, startTime=1732079836485; duration=0sec 2024-11-20T05:17:16,520 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:16,520 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A 2024-11-20T05:17:16,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T05:17:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:16,538 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:17:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:16,550 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:16,550 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/3b411a49ba3346b18268ac7658ac6835 is 50, key is test_row_0/C:col10/1732079834445/Put/seqid=0 2024-11-20T05:17:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112012e135fcd0a8485e9eb4cc82adf1a356_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079835652/Put/seqid=0 2024-11-20T05:17:16,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742403_1579 (size=12949) 2024-11-20T05:17:16,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742404_1580 (size=12454) 2024-11-20T05:17:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:16,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:16,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 413 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079896829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 469 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079896829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T05:17:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 414 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079896937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 470 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079896937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:16,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:16,972 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112012e135fcd0a8485e9eb4cc82adf1a356_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112012e135fcd0a8485e9eb4cc82adf1a356_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:16,972 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/3b411a49ba3346b18268ac7658ac6835 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/3b411a49ba3346b18268ac7658ac6835 2024-11-20T05:17:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/8306d821c8904b75b56a4507545bb3cf, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/8306d821c8904b75b56a4507545bb3cf is 175, key is test_row_0/A:col10/1732079835652/Put/seqid=0 2024-11-20T05:17:16,976 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 3b411a49ba3346b18268ac7658ac6835(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:16,976 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:16,977 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, priority=13, startTime=1732079836486; duration=0sec 2024-11-20T05:17:16,977 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:16,977 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C 2024-11-20T05:17:16,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742405_1581 (size=31255) 2024-11-20T05:17:17,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 415 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079897147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:17,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 471 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079897147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:17,382 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=891, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/8306d821c8904b75b56a4507545bb3cf 2024-11-20T05:17:17,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/1645e815089e4cf490a2a95466232529 is 50, key is test_row_0/B:col10/1732079835652/Put/seqid=0 2024-11-20T05:17:17,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742406_1582 (size=12301) 2024-11-20T05:17:17,391 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=891 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/1645e815089e4cf490a2a95466232529 2024-11-20T05:17:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a11ecf7cebb646029a3b91b1114a5d81 is 50, key is test_row_0/C:col10/1732079835652/Put/seqid=0 2024-11-20T05:17:17,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742407_1583 (size=12301) 2024-11-20T05:17:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 416 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079897457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 472 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079897457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:17,800 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=891 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a11ecf7cebb646029a3b91b1114a5d81 2024-11-20T05:17:17,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/8306d821c8904b75b56a4507545bb3cf as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/8306d821c8904b75b56a4507545bb3cf 2024-11-20T05:17:17,805 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/8306d821c8904b75b56a4507545bb3cf, entries=150, sequenceid=891, filesize=30.5 K 2024-11-20T05:17:17,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/1645e815089e4cf490a2a95466232529 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1645e815089e4cf490a2a95466232529 2024-11-20T05:17:17,808 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1645e815089e4cf490a2a95466232529, entries=150, sequenceid=891, filesize=12.0 K 2024-11-20T05:17:17,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/a11ecf7cebb646029a3b91b1114a5d81 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a11ecf7cebb646029a3b91b1114a5d81 2024-11-20T05:17:17,811 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a11ecf7cebb646029a3b91b1114a5d81, entries=150, sequenceid=891, filesize=12.0 K 2024-11-20T05:17:17,812 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1274ms, sequenceid=891, compaction requested=false 2024-11-20T05:17:17,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:17,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:17,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-20T05:17:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-20T05:17:17,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T05:17:17,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0520 sec 2024-11-20T05:17:17,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0570 sec 2024-11-20T05:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T05:17:17,917 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:17,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T05:17:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-20T05:17:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T05:17:17,919 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T05:17:17,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T05:17:17,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T05:17:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:17,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:17:17,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:17,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:17,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:17,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:17,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:17,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:17,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112053d28e0e90ae4b7ebecdc4c4e33a1e95_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079836814/Put/seqid=0 2024-11-20T05:17:17,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742408_1584 (size=12454) 2024-11-20T05:17:17,978 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:17,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112053d28e0e90ae4b7ebecdc4c4e33a1e95_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112053d28e0e90ae4b7ebecdc4c4e33a1e95_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:17,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/3337c15458234bbea7e7453a364553e1, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:17,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/3337c15458234bbea7e7453a364553e1 is 175, key is test_row_0/A:col10/1732079836814/Put/seqid=0 2024-11-20T05:17:17,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742409_1585 (size=31255) 2024-11-20T05:17:18,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 424 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079897998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 480 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079898000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T05:17:18,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T05:17:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 425 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079898107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 481 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079898107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T05:17:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T05:17:18,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 426 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079898317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 482 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079898318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T05:17:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T05:17:18,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=908, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/3337c15458234bbea7e7453a364553e1 2024-11-20T05:17:18,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/9073491af93b41bda82c58087238c959 is 50, key is test_row_0/B:col10/1732079836814/Put/seqid=0 2024-11-20T05:17:18,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742410_1586 (size=12301) 2024-11-20T05:17:18,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/9073491af93b41bda82c58087238c959 2024-11-20T05:17:18,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/52c8ece86140486a89aee0475975f553 is 50, key is test_row_0/C:col10/1732079836814/Put/seqid=0 2024-11-20T05:17:18,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742411_1587 (size=12301) 2024-11-20T05:17:18,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/52c8ece86140486a89aee0475975f553 2024-11-20T05:17:18,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/3337c15458234bbea7e7453a364553e1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3337c15458234bbea7e7453a364553e1 2024-11-20T05:17:18,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3337c15458234bbea7e7453a364553e1, entries=150, sequenceid=908, filesize=30.5 K 2024-11-20T05:17:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/9073491af93b41bda82c58087238c959 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/9073491af93b41bda82c58087238c959 2024-11-20T05:17:18,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/9073491af93b41bda82c58087238c959, entries=150, sequenceid=908, filesize=12.0 K 2024-11-20T05:17:18,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/52c8ece86140486a89aee0475975f553 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/52c8ece86140486a89aee0475975f553 2024-11-20T05:17:18,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/52c8ece86140486a89aee0475975f553, entries=150, sequenceid=908, filesize=12.0 K 2024-11-20T05:17:18,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 458ms, sequenceid=908, compaction requested=true 2024-11-20T05:17:18,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:18,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:17:18,427 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:18,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:18,427 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:18,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:17:18,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:18,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:17:18,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:18,428 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:18,428 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B is initiating minor compaction (all files) 2024-11-20T05:17:18,428 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,428 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/bb83438266404caab2314d174b4be764, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1645e815089e4cf490a2a95466232529, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/9073491af93b41bda82c58087238c959] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=36.7 K 2024-11-20T05:17:18,429 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:18,429 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting bb83438266404caab2314d174b4be764, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1732079834445 2024-11-20T05:17:18,429 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A is initiating minor compaction (all files) 2024-11-20T05:17:18,429 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,429 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/09821cdd622a423e92d02bf1d1381289, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/8306d821c8904b75b56a4507545bb3cf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3337c15458234bbea7e7453a364553e1] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=92.2 K 2024-11-20T05:17:18,429 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,429 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/09821cdd622a423e92d02bf1d1381289, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/8306d821c8904b75b56a4507545bb3cf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3337c15458234bbea7e7453a364553e1] 2024-11-20T05:17:18,429 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1645e815089e4cf490a2a95466232529, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=891, earliestPutTs=1732079835640 2024-11-20T05:17:18,430 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09821cdd622a423e92d02bf1d1381289, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1732079834445 2024-11-20T05:17:18,430 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9073491af93b41bda82c58087238c959, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1732079836814 2024-11-20T05:17:18,430 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8306d821c8904b75b56a4507545bb3cf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=891, earliestPutTs=1732079835640 2024-11-20T05:17:18,431 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3337c15458234bbea7e7453a364553e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1732079836814 2024-11-20T05:17:18,436 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:18,442 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:18,443 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/3d2f8f70fa034bcb9475e27cf32fb011 is 50, key is test_row_0/B:col10/1732079836814/Put/seqid=0 2024-11-20T05:17:18,443 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120af580a54babc49308ed8c84b11983835_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:18,445 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120af580a54babc49308ed8c84b11983835_ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:18,445 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120af580a54babc49308ed8c84b11983835_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:18,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742412_1588 (size=13051) 2024-11-20T05:17:18,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742413_1589 (size=4469) 2024-11-20T05:17:18,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T05:17:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,530 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T05:17:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203c2d08d663c6466b8245fa3af5216353_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079837983/Put/seqid=0 2024-11-20T05:17:18,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742414_1590 (size=12454) 2024-11-20T05:17:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T05:17:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. as already flushing 2024-11-20T05:17:18,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079898592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079898595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56510 deadline: 1732079898597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,602 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_2' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 427 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079898627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 483 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079898627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079898707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079898708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,812 INFO [Thread-2261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2261. 2024-11-20T05:17:18,812 DEBUG [Thread-2261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:18,812 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,812 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:18,814 INFO [Thread-2257 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2257. 2024-11-20T05:17:18,814 DEBUG [Thread-2257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:18,814 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,815 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:18,815 INFO [Thread-2259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2259. 2024-11-20T05:17:18,815 DEBUG [Thread-2259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:18,815 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,816 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:18,817 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-11-20T05:17:18,817 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:18,817 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,817 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:18,819 INFO [Thread-2263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2263. 2024-11-20T05:17:18,819 DEBUG [Thread-2263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:18,819 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,819 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:18,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:18,856 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/3d2f8f70fa034bcb9475e27cf32fb011 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3d2f8f70fa034bcb9475e27cf32fb011 2024-11-20T05:17:18,860 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 3d2f8f70fa034bcb9475e27cf32fb011(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:18,860 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:18,860 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, priority=13, startTime=1732079838427; duration=0sec 2024-11-20T05:17:18,860 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:18,860 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B 2024-11-20T05:17:18,860 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:18,861 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:18,861 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C is initiating minor compaction (all files) 2024-11-20T05:17:18,861 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:18,861 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/3b411a49ba3346b18268ac7658ac6835, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a11ecf7cebb646029a3b91b1114a5d81, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/52c8ece86140486a89aee0475975f553] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=36.7 K 2024-11-20T05:17:18,861 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b411a49ba3346b18268ac7658ac6835, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1732079834445 2024-11-20T05:17:18,862 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting a11ecf7cebb646029a3b91b1114a5d81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=891, earliestPutTs=1732079835640 2024-11-20T05:17:18,862 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#A#compaction#491 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:18,862 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 52c8ece86140486a89aee0475975f553, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1732079836814 2024-11-20T05:17:18,863 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/be39d4898892431ead10b7a054e823e5 is 175, key is test_row_0/A:col10/1732079836814/Put/seqid=0 2024-11-20T05:17:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742415_1591 (size=32005) 2024-11-20T05:17:18,869 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:18,869 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/5d412999fc1c4734a8b575e977a75e28 is 50, key is test_row_0/C:col10/1732079836814/Put/seqid=0 2024-11-20T05:17:18,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742416_1592 (size=13051) 2024-11-20T05:17:18,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079898918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079898918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:18,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:18,951 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203c2d08d663c6466b8245fa3af5216353_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203c2d08d663c6466b8245fa3af5216353_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9ab555bd279e41308fb02723e490bc78, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:18,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9ab555bd279e41308fb02723e490bc78 is 175, key is test_row_0/A:col10/1732079837983/Put/seqid=0 2024-11-20T05:17:18,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742417_1593 (size=31255) 2024-11-20T05:17:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T05:17:19,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:19,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 484 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079899137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 428 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079899137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079899227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079899227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:19,270 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/be39d4898892431ead10b7a054e823e5 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/be39d4898892431ead10b7a054e823e5 2024-11-20T05:17:19,274 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into be39d4898892431ead10b7a054e823e5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:19,274 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:19,274 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, priority=13, startTime=1732079838427; duration=0sec 2024-11-20T05:17:19,274 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:19,274 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A 2024-11-20T05:17:19,275 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/5d412999fc1c4734a8b575e977a75e28 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/5d412999fc1c4734a8b575e977a75e28 2024-11-20T05:17:19,279 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into 5d412999fc1c4734a8b575e977a75e28(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:19,279 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:19,279 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, priority=13, startTime=1732079838427; duration=0sec 2024-11-20T05:17:19,279 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:19,279 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C 2024-11-20T05:17:19,357 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=924, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9ab555bd279e41308fb02723e490bc78 2024-11-20T05:17:19,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/68aea7369a0c4f7baf5d8c6fb3efb3e8 is 50, key is test_row_0/B:col10/1732079837983/Put/seqid=0 2024-11-20T05:17:19,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742418_1594 (size=12301) 2024-11-20T05:17:19,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56546 deadline: 1732079899738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56566 deadline: 1732079899738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:19,772 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=924 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/68aea7369a0c4f7baf5d8c6fb3efb3e8 2024-11-20T05:17:19,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8f347cc4872743989e1e367d610e72df is 50, key is test_row_0/C:col10/1732079837983/Put/seqid=0 2024-11-20T05:17:19,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742419_1595 (size=12301) 2024-11-20T05:17:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T05:17:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T05:17:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 485 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56526 deadline: 1732079900147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] ipc.CallRunner(138): callId: 429 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56512 deadline: 1732079900147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:20,148 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:20,148 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 51d5d2011a1c,44753,1732079655401 for 'test_row_1' in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T05:17:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 , the old value is region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T05:17:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., hostname=51d5d2011a1c,44753,1732079655401, seqNum=5 because the exception is null or not the one we care about 2024-11-20T05:17:20,192 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=924 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8f347cc4872743989e1e367d610e72df 2024-11-20T05:17:20,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/9ab555bd279e41308fb02723e490bc78 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9ab555bd279e41308fb02723e490bc78 2024-11-20T05:17:20,208 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9ab555bd279e41308fb02723e490bc78, entries=150, sequenceid=924, filesize=30.5 K 2024-11-20T05:17:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/68aea7369a0c4f7baf5d8c6fb3efb3e8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/68aea7369a0c4f7baf5d8c6fb3efb3e8 2024-11-20T05:17:20,214 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/68aea7369a0c4f7baf5d8c6fb3efb3e8, entries=150, sequenceid=924, filesize=12.0 K 2024-11-20T05:17:20,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/8f347cc4872743989e1e367d610e72df as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8f347cc4872743989e1e367d610e72df 2024-11-20T05:17:20,217 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8f347cc4872743989e1e367d610e72df, entries=150, sequenceid=924, filesize=12.0 K 2024-11-20T05:17:20,218 INFO [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1688ms, sequenceid=924, compaction requested=false 2024-11-20T05:17:20,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:20,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:20,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51d5d2011a1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-20T05:17:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-20T05:17:20,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-20T05:17:20,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2990 sec 2024-11-20T05:17:20,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.3030 sec 2024-11-20T05:17:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44753 {}] regionserver.HRegion(8855): Flush requested on ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:20,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T05:17:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:20,748 INFO [Thread-2248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2248. 2024-11-20T05:17:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:20,748 DEBUG [Thread-2248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:20,748 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:20,748 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:20,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:20,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d6645703ad6649918d73339315f97afb_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_0/A:col10/1732079838594/Put/seqid=0 2024-11-20T05:17:20,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742420_1596 (size=12454) 2024-11-20T05:17:20,758 INFO [Thread-2254 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2254. 2024-11-20T05:17:20,758 DEBUG [Thread-2254 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:20,758 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:20,758 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:20,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:21,157 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:21,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d6645703ad6649918d73339315f97afb_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d6645703ad6649918d73339315f97afb_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:21,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d57f60ca2c7c4db6929d1ad60aa59eae, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:21,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d57f60ca2c7c4db6929d1ad60aa59eae is 175, key is test_row_0/A:col10/1732079838594/Put/seqid=0 2024-11-20T05:17:21,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742421_1597 (size=31255) 2024-11-20T05:17:21,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=944, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d57f60ca2c7c4db6929d1ad60aa59eae 2024-11-20T05:17:21,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/3705d356008e4e299dbd2cc7ca5acf68 is 50, key is test_row_0/B:col10/1732079838594/Put/seqid=0 2024-11-20T05:17:21,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742422_1598 (size=12301) 2024-11-20T05:17:21,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=944 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/3705d356008e4e299dbd2cc7ca5acf68 2024-11-20T05:17:21,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/4d986f56c08142eebd765a59bad14c33 is 50, key is test_row_0/C:col10/1732079838594/Put/seqid=0 2024-11-20T05:17:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742423_1599 (size=12301) 2024-11-20T05:17:21,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=944 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/4d986f56c08142eebd765a59bad14c33 2024-11-20T05:17:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/d57f60ca2c7c4db6929d1ad60aa59eae as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d57f60ca2c7c4db6929d1ad60aa59eae 2024-11-20T05:17:22,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d57f60ca2c7c4db6929d1ad60aa59eae, entries=150, sequenceid=944, filesize=30.5 K 2024-11-20T05:17:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/3705d356008e4e299dbd2cc7ca5acf68 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3705d356008e4e299dbd2cc7ca5acf68 2024-11-20T05:17:22,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3705d356008e4e299dbd2cc7ca5acf68, entries=150, sequenceid=944, filesize=12.0 K 2024-11-20T05:17:22,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/4d986f56c08142eebd765a59bad14c33 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/4d986f56c08142eebd765a59bad14c33 2024-11-20T05:17:22,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/4d986f56c08142eebd765a59bad14c33, entries=150, sequenceid=944, filesize=12.0 K 2024-11-20T05:17:22,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1264ms, sequenceid=944, compaction requested=true 2024-11-20T05:17:22,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:22,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T05:17:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:22,013 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T05:17:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T05:17:22,013 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:22,013 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:22,013 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:22,013 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B is initiating minor compaction (all files) 2024-11-20T05:17:22,013 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A is initiating minor compaction (all files) 2024-11-20T05:17:22,013 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:22,013 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:22,014 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3d2f8f70fa034bcb9475e27cf32fb011, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/68aea7369a0c4f7baf5d8c6fb3efb3e8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3705d356008e4e299dbd2cc7ca5acf68] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=36.8 K 2024-11-20T05:17:22,014 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/be39d4898892431ead10b7a054e823e5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9ab555bd279e41308fb02723e490bc78, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d57f60ca2c7c4db6929d1ad60aa59eae] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=92.3 K 2024-11-20T05:17:22,014 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:22,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. files: [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/be39d4898892431ead10b7a054e823e5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9ab555bd279e41308fb02723e490bc78, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d57f60ca2c7c4db6929d1ad60aa59eae] 2024-11-20T05:17:22,014 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2f8f70fa034bcb9475e27cf32fb011, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1732079836814 2024-11-20T05:17:22,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting be39d4898892431ead10b7a054e823e5, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1732079836814 2024-11-20T05:17:22,014 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 68aea7369a0c4f7baf5d8c6fb3efb3e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=924, earliestPutTs=1732079837983 2024-11-20T05:17:22,014 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ab555bd279e41308fb02723e490bc78, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=924, earliestPutTs=1732079837983 2024-11-20T05:17:22,015 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3705d356008e4e299dbd2cc7ca5acf68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=944, earliestPutTs=1732079838585 2024-11-20T05:17:22,015 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d57f60ca2c7c4db6929d1ad60aa59eae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=944, earliestPutTs=1732079838585 2024-11-20T05:17:22,024 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:22,025 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ce6980c60ab1458fa277de73279858a4_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:22,027 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:22,027 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/a1fc90d77a5240ccbfd0c8045c88e9f1 is 50, key is test_row_0/B:col10/1732079838594/Put/seqid=0 2024-11-20T05:17:22,027 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ce6980c60ab1458fa277de73279858a4_ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:22,028 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ce6980c60ab1458fa277de73279858a4_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:22,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742425_1601 (size=4469) 2024-11-20T05:17:22,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742424_1600 (size=13153) 2024-11-20T05:17:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T05:17:22,088 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:22,181 INFO [Thread-2246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2246. 2024-11-20T05:17:22,181 INFO [Thread-2252 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2252. 2024-11-20T05:17:22,181 DEBUG [Thread-2246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:22,181 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:22,181 DEBUG [Thread-2252 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:22,181 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:22,181 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:22,181 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:22,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:22,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:22,432 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#A#compaction#500 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:22,433 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a85aca44d9274dba9a6e02f28d3a609d is 175, key is test_row_0/A:col10/1732079838594/Put/seqid=0 2024-11-20T05:17:22,436 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/a1fc90d77a5240ccbfd0c8045c88e9f1 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a1fc90d77a5240ccbfd0c8045c88e9f1 2024-11-20T05:17:22,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742426_1602 (size=32107) 2024-11-20T05:17:22,439 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into a1fc90d77a5240ccbfd0c8045c88e9f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:22,439 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:22,439 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, priority=13, startTime=1732079842013; duration=0sec 2024-11-20T05:17:22,439 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T05:17:22,439 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:B 2024-11-20T05:17:22,439 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T05:17:22,440 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T05:17:22,440 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1541): ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C is initiating minor compaction (all files) 2024-11-20T05:17:22,440 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C in TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:22,440 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/5d412999fc1c4734a8b575e977a75e28, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8f347cc4872743989e1e367d610e72df, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/4d986f56c08142eebd765a59bad14c33] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp, totalSize=36.8 K 2024-11-20T05:17:22,440 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d412999fc1c4734a8b575e977a75e28, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1732079836814 2024-11-20T05:17:22,440 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f347cc4872743989e1e367d610e72df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=924, earliestPutTs=1732079837983 2024-11-20T05:17:22,441 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d986f56c08142eebd765a59bad14c33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=944, earliestPutTs=1732079838585 2024-11-20T05:17:22,446 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee51cd6a06f90dcb79cbe1ee3eb2fbd4#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T05:17:22,447 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/e5fecc96e0d640568d06a5eed372e063 is 50, key is test_row_0/C:col10/1732079838594/Put/seqid=0 2024-11-20T05:17:22,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742427_1603 (size=13153) 2024-11-20T05:17:22,842 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/a85aca44d9274dba9a6e02f28d3a609d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a85aca44d9274dba9a6e02f28d3a609d 2024-11-20T05:17:22,848 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into a85aca44d9274dba9a6e02f28d3a609d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:22,848 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:22,848 INFO [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, priority=13, startTime=1732079842012; duration=0sec 2024-11-20T05:17:22,848 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:22,848 DEBUG [RS:0;51d5d2011a1c:44753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:A 2024-11-20T05:17:22,854 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/e5fecc96e0d640568d06a5eed372e063 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e5fecc96e0d640568d06a5eed372e063 2024-11-20T05:17:22,858 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C of ee51cd6a06f90dcb79cbe1ee3eb2fbd4 into e5fecc96e0d640568d06a5eed372e063(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T05:17:22,858 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: 2024-11-20T05:17:22,858 INFO [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., storeName=ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, priority=13, startTime=1732079842013; duration=0sec 2024-11-20T05:17:22,858 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T05:17:22,858 DEBUG [RS:0;51d5d2011a1c:44753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee51cd6a06f90dcb79cbe1ee3eb2fbd4:C 2024-11-20T05:17:28,638 INFO [Thread-2250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2250. 2024-11-20T05:17:28,638 DEBUG [Thread-2250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T05:17:28,638 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:28,638 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:28,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 368 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 427 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2743 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2596 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2635 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2752 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2611 2024-11-20T05:17:28,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T05:17:28,638 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:17:28,639 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:17:28,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:28,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:28,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:28,639 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T05:17:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T05:17:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T05:17:28,642 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079848641"}]},"ts":"1732079848641"} 2024-11-20T05:17:28,643 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T05:17:28,643 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T05:17:28,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T05:17:28,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, UNASSIGN}] 2024-11-20T05:17:28,645 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, UNASSIGN 2024-11-20T05:17:28,646 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=CLOSING, regionLocation=51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:28,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, UNASSIGN because future has completed 2024-11-20T05:17:28,647 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T05:17:28,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401}] 2024-11-20T05:17:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T05:17:28,799 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:28,799 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T05:17:28,799 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing ee51cd6a06f90dcb79cbe1ee3eb2fbd4, disabling compactions & flushes 2024-11-20T05:17:28,799 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:28,799 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:28,799 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. after waiting 0 ms 2024-11-20T05:17:28,799 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:28,799 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T05:17:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=A 2024-11-20T05:17:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=B 2024-11-20T05:17:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee51cd6a06f90dcb79cbe1ee3eb2fbd4, store=C 2024-11-20T05:17:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T05:17:28,804 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112096939bcc156b4d03b4389c4c9a65cdf3_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 is 50, key is test_row_1/A:col10/1732079842169/Put/seqid=0 2024-11-20T05:17:28,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742428_1604 (size=9914) 2024-11-20T05:17:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T05:17:29,208 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T05:17:29,211 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112096939bcc156b4d03b4389c4c9a65cdf3_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112096939bcc156b4d03b4389c4c9a65cdf3_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:29,213 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/c38a4295376d49069faac53c4cee76c8, store: [table=TestAcidGuarantees family=A region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4] 2024-11-20T05:17:29,214 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/c38a4295376d49069faac53c4cee76c8 is 175, key is test_row_1/A:col10/1732079842169/Put/seqid=0 2024-11-20T05:17:29,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742429_1605 (size=22561) 2024-11-20T05:17:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T05:17:29,618 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=954, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/c38a4295376d49069faac53c4cee76c8 2024-11-20T05:17:29,627 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/de574a6b10f242b68ec83c284afe965a is 50, key is test_row_1/B:col10/1732079842169/Put/seqid=0 2024-11-20T05:17:29,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742430_1606 (size=9857) 2024-11-20T05:17:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T05:17:30,031 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=954 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/de574a6b10f242b68ec83c284afe965a 2024-11-20T05:17:30,042 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/813b7b5e6d454f4d93228db5df98c44b is 50, key is test_row_1/C:col10/1732079842169/Put/seqid=0 2024-11-20T05:17:30,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742431_1607 (size=9857) 2024-11-20T05:17:30,446 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=954 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/813b7b5e6d454f4d93228db5df98c44b 2024-11-20T05:17:30,451 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/A/c38a4295376d49069faac53c4cee76c8 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/c38a4295376d49069faac53c4cee76c8 2024-11-20T05:17:30,454 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/c38a4295376d49069faac53c4cee76c8, entries=100, sequenceid=954, filesize=22.0 K 2024-11-20T05:17:30,455 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/B/de574a6b10f242b68ec83c284afe965a as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de574a6b10f242b68ec83c284afe965a 2024-11-20T05:17:30,458 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de574a6b10f242b68ec83c284afe965a, entries=100, sequenceid=954, filesize=9.6 K 2024-11-20T05:17:30,459 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/.tmp/C/813b7b5e6d454f4d93228db5df98c44b as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/813b7b5e6d454f4d93228db5df98c44b 2024-11-20T05:17:30,462 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/813b7b5e6d454f4d93228db5df98c44b, entries=100, sequenceid=954, filesize=9.6 K 2024-11-20T05:17:30,463 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1664ms, sequenceid=954, compaction requested=false 2024-11-20T05:17:30,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/7368636bdfea4253aa9e8f3dc287471b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9101b9f866d74b869aceb5c6998ca692, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/952742d697804ba4b0adf2de0e7013cc, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d4b313108a79496396298f1505479e2a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/b5209ec6d86a48bdad2b536da04def9c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a2224a0c332e4a20a826d742caa3d629, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cc0db878a76e4f5dbd6332098e7e97c0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/45f93d78acc84b9390e8e43abb5dd2be, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/ed0f46e3c0f545778444119bb5aac520, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a48f62586bf043e39faf3d730a6cf6cb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/79f73f38809c45cdb984e6dbe2f7919c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/213f567131e74d048368b4d06d0313a4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/563cdb729e5b46dcb28127443eac6d0f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/220b291ccdea4534b58dac07e935639b, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/895f0dbdf17b4e4f895fc60cdbc97e89, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/6248505bd44e4240a92b81bebb52e57e, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3ae591557f1a4f86afb1e4995fa46a07, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cb1530c9621f441e96572f4b45c2aebe, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/09821cdd622a423e92d02bf1d1381289, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/24075bf038a74ccebc390ef72795ce38, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/8306d821c8904b75b56a4507545bb3cf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/be39d4898892431ead10b7a054e823e5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3337c15458234bbea7e7453a364553e1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9ab555bd279e41308fb02723e490bc78, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d57f60ca2c7c4db6929d1ad60aa59eae] to archive 2024-11-20T05:17:30,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:17:30,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/7368636bdfea4253aa9e8f3dc287471b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/7368636bdfea4253aa9e8f3dc287471b 2024-11-20T05:17:30,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9101b9f866d74b869aceb5c6998ca692 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9101b9f866d74b869aceb5c6998ca692 2024-11-20T05:17:30,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/952742d697804ba4b0adf2de0e7013cc to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/952742d697804ba4b0adf2de0e7013cc 2024-11-20T05:17:30,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d4b313108a79496396298f1505479e2a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d4b313108a79496396298f1505479e2a 2024-11-20T05:17:30,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/b5209ec6d86a48bdad2b536da04def9c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/b5209ec6d86a48bdad2b536da04def9c 2024-11-20T05:17:30,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a2224a0c332e4a20a826d742caa3d629 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a2224a0c332e4a20a826d742caa3d629 2024-11-20T05:17:30,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cc0db878a76e4f5dbd6332098e7e97c0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cc0db878a76e4f5dbd6332098e7e97c0 2024-11-20T05:17:30,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/45f93d78acc84b9390e8e43abb5dd2be to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/45f93d78acc84b9390e8e43abb5dd2be 2024-11-20T05:17:30,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/ed0f46e3c0f545778444119bb5aac520 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/ed0f46e3c0f545778444119bb5aac520 2024-11-20T05:17:30,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a48f62586bf043e39faf3d730a6cf6cb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a48f62586bf043e39faf3d730a6cf6cb 2024-11-20T05:17:30,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/79f73f38809c45cdb984e6dbe2f7919c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/79f73f38809c45cdb984e6dbe2f7919c 2024-11-20T05:17:30,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/213f567131e74d048368b4d06d0313a4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/213f567131e74d048368b4d06d0313a4 2024-11-20T05:17:30,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/563cdb729e5b46dcb28127443eac6d0f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/563cdb729e5b46dcb28127443eac6d0f 2024-11-20T05:17:30,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/220b291ccdea4534b58dac07e935639b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/220b291ccdea4534b58dac07e935639b 2024-11-20T05:17:30,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/895f0dbdf17b4e4f895fc60cdbc97e89 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/895f0dbdf17b4e4f895fc60cdbc97e89 2024-11-20T05:17:30,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/6248505bd44e4240a92b81bebb52e57e to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/6248505bd44e4240a92b81bebb52e57e 2024-11-20T05:17:30,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3ae591557f1a4f86afb1e4995fa46a07 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3ae591557f1a4f86afb1e4995fa46a07 2024-11-20T05:17:30,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cb1530c9621f441e96572f4b45c2aebe to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/cb1530c9621f441e96572f4b45c2aebe 2024-11-20T05:17:30,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/09821cdd622a423e92d02bf1d1381289 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/09821cdd622a423e92d02bf1d1381289 2024-11-20T05:17:30,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/24075bf038a74ccebc390ef72795ce38 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/24075bf038a74ccebc390ef72795ce38 2024-11-20T05:17:30,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/8306d821c8904b75b56a4507545bb3cf to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/8306d821c8904b75b56a4507545bb3cf 2024-11-20T05:17:30,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/be39d4898892431ead10b7a054e823e5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/be39d4898892431ead10b7a054e823e5 2024-11-20T05:17:30,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3337c15458234bbea7e7453a364553e1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/3337c15458234bbea7e7453a364553e1 2024-11-20T05:17:30,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9ab555bd279e41308fb02723e490bc78 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/9ab555bd279e41308fb02723e490bc78 2024-11-20T05:17:30,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d57f60ca2c7c4db6929d1ad60aa59eae to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/d57f60ca2c7c4db6929d1ad60aa59eae 2024-11-20T05:17:30,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/58472683320d4a31970667da0fdc6f31, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8991d25e4ba440b6a7273667b9ec4ae7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/07a3b3a8adf34737a1afbdaff24bd7b8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de818a63b80f4eba9faf12d0d184ea49, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/4ce9abc6b2904293a3287905ce1d47bf, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/abaebafeacb64315b0c1ffdb122d24bb, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/e5e8d5aff3224e20a25fdbff8e1be534, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4754598a790409e8e588f7d0daa183f, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3f75e489838e4414ab8ca91190192fe1, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/560048235c124d49bcb7b4add730e01c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a3ef6abe92bb44a6947e96f1f552a6ae, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4d11e5bd9a9465da7ac4b86604036fa, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/01428b2cda2342009b91e0d297c946c2, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1cab20ecdabe4f569f8af4284ef3fbb5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8aa30da516b8440ca1e5fec21862e970, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/aad7b1be6e5d4bfd904d8aba1ca4606d, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/5b2309d677bd428395a36bfcc16375e3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/6d97fee72cad4502884ec2cdcd19a2f5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/bb83438266404caab2314d174b4be764, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/b14869023ac3440cab8276c4be2c62d3, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1645e815089e4cf490a2a95466232529, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3d2f8f70fa034bcb9475e27cf32fb011, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/9073491af93b41bda82c58087238c959, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/68aea7369a0c4f7baf5d8c6fb3efb3e8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3705d356008e4e299dbd2cc7ca5acf68] to archive 2024-11-20T05:17:30,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:17:30,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/58472683320d4a31970667da0fdc6f31 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/58472683320d4a31970667da0fdc6f31 2024-11-20T05:17:30,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8991d25e4ba440b6a7273667b9ec4ae7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8991d25e4ba440b6a7273667b9ec4ae7 2024-11-20T05:17:30,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/07a3b3a8adf34737a1afbdaff24bd7b8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/07a3b3a8adf34737a1afbdaff24bd7b8 2024-11-20T05:17:30,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de818a63b80f4eba9faf12d0d184ea49 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de818a63b80f4eba9faf12d0d184ea49 2024-11-20T05:17:30,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/4ce9abc6b2904293a3287905ce1d47bf to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/4ce9abc6b2904293a3287905ce1d47bf 2024-11-20T05:17:30,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/abaebafeacb64315b0c1ffdb122d24bb to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/abaebafeacb64315b0c1ffdb122d24bb 2024-11-20T05:17:30,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/e5e8d5aff3224e20a25fdbff8e1be534 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/e5e8d5aff3224e20a25fdbff8e1be534 2024-11-20T05:17:30,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4754598a790409e8e588f7d0daa183f to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4754598a790409e8e588f7d0daa183f 2024-11-20T05:17:30,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3f75e489838e4414ab8ca91190192fe1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3f75e489838e4414ab8ca91190192fe1 2024-11-20T05:17:30,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/560048235c124d49bcb7b4add730e01c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/560048235c124d49bcb7b4add730e01c 2024-11-20T05:17:30,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a3ef6abe92bb44a6947e96f1f552a6ae to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a3ef6abe92bb44a6947e96f1f552a6ae 2024-11-20T05:17:30,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4d11e5bd9a9465da7ac4b86604036fa to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/f4d11e5bd9a9465da7ac4b86604036fa 2024-11-20T05:17:30,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/01428b2cda2342009b91e0d297c946c2 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/01428b2cda2342009b91e0d297c946c2 2024-11-20T05:17:30,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1cab20ecdabe4f569f8af4284ef3fbb5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1cab20ecdabe4f569f8af4284ef3fbb5 2024-11-20T05:17:30,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8aa30da516b8440ca1e5fec21862e970 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/8aa30da516b8440ca1e5fec21862e970 2024-11-20T05:17:30,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/aad7b1be6e5d4bfd904d8aba1ca4606d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/aad7b1be6e5d4bfd904d8aba1ca4606d 2024-11-20T05:17:30,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/5b2309d677bd428395a36bfcc16375e3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/5b2309d677bd428395a36bfcc16375e3 2024-11-20T05:17:30,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/6d97fee72cad4502884ec2cdcd19a2f5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/6d97fee72cad4502884ec2cdcd19a2f5 2024-11-20T05:17:30,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/bb83438266404caab2314d174b4be764 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/bb83438266404caab2314d174b4be764 2024-11-20T05:17:30,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/b14869023ac3440cab8276c4be2c62d3 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/b14869023ac3440cab8276c4be2c62d3 2024-11-20T05:17:30,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1645e815089e4cf490a2a95466232529 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/1645e815089e4cf490a2a95466232529 2024-11-20T05:17:30,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3d2f8f70fa034bcb9475e27cf32fb011 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3d2f8f70fa034bcb9475e27cf32fb011 2024-11-20T05:17:30,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/9073491af93b41bda82c58087238c959 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/9073491af93b41bda82c58087238c959 2024-11-20T05:17:30,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/68aea7369a0c4f7baf5d8c6fb3efb3e8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/68aea7369a0c4f7baf5d8c6fb3efb3e8 2024-11-20T05:17:30,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3705d356008e4e299dbd2cc7ca5acf68 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/3705d356008e4e299dbd2cc7ca5acf68 2024-11-20T05:17:30,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8000049651aa424396db5bb6c4177139, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/1eda432395534b5fb52846cdf058b8a4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/af8517fe427e429fb0f56c9312f1e229, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0a0ddb32b5e14e5eb16e9239de0c7df5, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/79a30cf094f44f02be8d91371724c5e0, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/6c14e24161d54a0db44587610cd95f0c, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e9d228c62cd247b995037ff699cef550, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d0766cbbe2eb4c0e80a8934b2f9e4367, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/f02317817e224e939975060bac9e21d8, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ac7faa734d0e44169c119a040e2374ab, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/fdbebcd60e1c4afbb2721f6c32e79922, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a293929d6b1444ad9d66e393a85388f7, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a803fba86cc1444daf00a66af82b6f7a, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0cc54b785aa944a896a5ae9aa30e19f4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d79ff5f99e5843579876de6e444410b4, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e0dc2815a8774955bd12a732db1b7c14, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ba27157a090642a593a169aed7b129de, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/2d23047d1a1b4e4798fbee9b63ceef89, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/3b411a49ba3346b18268ac7658ac6835, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8e627bac46a748d698f2332fecba18de, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a11ecf7cebb646029a3b91b1114a5d81, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/5d412999fc1c4734a8b575e977a75e28, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/52c8ece86140486a89aee0475975f553, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8f347cc4872743989e1e367d610e72df, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/4d986f56c08142eebd765a59bad14c33] to archive 2024-11-20T05:17:30,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T05:17:30,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8000049651aa424396db5bb6c4177139 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8000049651aa424396db5bb6c4177139 2024-11-20T05:17:30,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/1eda432395534b5fb52846cdf058b8a4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/1eda432395534b5fb52846cdf058b8a4 2024-11-20T05:17:30,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/af8517fe427e429fb0f56c9312f1e229 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/af8517fe427e429fb0f56c9312f1e229 2024-11-20T05:17:30,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0a0ddb32b5e14e5eb16e9239de0c7df5 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0a0ddb32b5e14e5eb16e9239de0c7df5 2024-11-20T05:17:30,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/79a30cf094f44f02be8d91371724c5e0 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/79a30cf094f44f02be8d91371724c5e0 2024-11-20T05:17:30,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/6c14e24161d54a0db44587610cd95f0c to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/6c14e24161d54a0db44587610cd95f0c 2024-11-20T05:17:30,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e9d228c62cd247b995037ff699cef550 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e9d228c62cd247b995037ff699cef550 2024-11-20T05:17:30,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d0766cbbe2eb4c0e80a8934b2f9e4367 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d0766cbbe2eb4c0e80a8934b2f9e4367 2024-11-20T05:17:30,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/f02317817e224e939975060bac9e21d8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/f02317817e224e939975060bac9e21d8 2024-11-20T05:17:30,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ac7faa734d0e44169c119a040e2374ab to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ac7faa734d0e44169c119a040e2374ab 2024-11-20T05:17:30,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/fdbebcd60e1c4afbb2721f6c32e79922 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/fdbebcd60e1c4afbb2721f6c32e79922 2024-11-20T05:17:30,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a293929d6b1444ad9d66e393a85388f7 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a293929d6b1444ad9d66e393a85388f7 2024-11-20T05:17:30,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a803fba86cc1444daf00a66af82b6f7a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a803fba86cc1444daf00a66af82b6f7a 2024-11-20T05:17:30,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0cc54b785aa944a896a5ae9aa30e19f4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/0cc54b785aa944a896a5ae9aa30e19f4 2024-11-20T05:17:30,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d79ff5f99e5843579876de6e444410b4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/d79ff5f99e5843579876de6e444410b4 2024-11-20T05:17:30,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e0dc2815a8774955bd12a732db1b7c14 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e0dc2815a8774955bd12a732db1b7c14 2024-11-20T05:17:30,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ba27157a090642a593a169aed7b129de to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/ba27157a090642a593a169aed7b129de 2024-11-20T05:17:30,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/2d23047d1a1b4e4798fbee9b63ceef89 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/2d23047d1a1b4e4798fbee9b63ceef89 2024-11-20T05:17:30,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/3b411a49ba3346b18268ac7658ac6835 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/3b411a49ba3346b18268ac7658ac6835 2024-11-20T05:17:30,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8e627bac46a748d698f2332fecba18de to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8e627bac46a748d698f2332fecba18de 2024-11-20T05:17:30,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a11ecf7cebb646029a3b91b1114a5d81 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/a11ecf7cebb646029a3b91b1114a5d81 2024-11-20T05:17:30,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/5d412999fc1c4734a8b575e977a75e28 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/5d412999fc1c4734a8b575e977a75e28 2024-11-20T05:17:30,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/52c8ece86140486a89aee0475975f553 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/52c8ece86140486a89aee0475975f553 2024-11-20T05:17:30,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8f347cc4872743989e1e367d610e72df to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/8f347cc4872743989e1e367d610e72df 2024-11-20T05:17:30,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/4d986f56c08142eebd765a59bad14c33 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/4d986f56c08142eebd765a59bad14c33 2024-11-20T05:17:30,548 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/recovered.edits/957.seqid, newMaxSeqId=957, maxSeqId=4 2024-11-20T05:17:30,550 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. 2024-11-20T05:17:30,550 DEBUG [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for ee51cd6a06f90dcb79cbe1ee3eb2fbd4: Waiting for close lock at 1732079848799Running coprocessor pre-close hooks at 1732079848799Disabling compacts and flushes for region at 1732079848799Disabling writes for close at 1732079848799Obtaining lock to block concurrent updates at 1732079848799Preparing flush snapshotting stores in ee51cd6a06f90dcb79cbe1ee3eb2fbd4 at 1732079848799Finished memstore snapshotting TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732079848800 (+1 ms)Flushing stores of TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4. at 1732079848800Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A: creating writer at 1732079848800Flushing mob file ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A: appending metadata at 1732079848804 (+4 ms)Flushing mob file ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A: closing flushed file at 1732079848804Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A: appending metadata at 1732079849213 (+409 ms)Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A: closing flushed file at 1732079849213Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B: creating writer at 1732079849625 (+412 ms)Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B: appending metadata at 1732079849627 (+2 ms)Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B: closing flushed file at 1732079849627Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C: creating writer at 1732079850038 (+411 ms)Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C: appending metadata at 1732079850041 (+3 ms)Flushing ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C: closing flushed file at 1732079850041Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c5cfb68: reopening flushed file at 1732079850451 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7502be5a: reopening flushed file at 1732079850455 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15a91662: reopening flushed file at 1732079850459 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ee51cd6a06f90dcb79cbe1ee3eb2fbd4 in 1664ms, sequenceid=954, compaction requested=false at 1732079850463 (+4 ms)Writing region close event to WAL at 1732079850543 (+80 ms)Running coprocessor post-close hooks at 1732079850550 (+7 ms)Closed at 1732079850550 2024-11-20T05:17:30,553 INFO [RS_CLOSE_REGION-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,553 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, regionState=CLOSED 2024-11-20T05:17:30,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 because future has completed 2024-11-20T05:17:30,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-20T05:17:30,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure ee51cd6a06f90dcb79cbe1ee3eb2fbd4, server=51d5d2011a1c,44753,1732079655401 in 1.9080 sec 2024-11-20T05:17:30,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-20T05:17:30,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee51cd6a06f90dcb79cbe1ee3eb2fbd4, UNASSIGN in 1.9120 sec 2024-11-20T05:17:30,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T05:17:30,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9150 sec 2024-11-20T05:17:30,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732079850560"}]},"ts":"1732079850560"} 2024-11-20T05:17:30,561 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T05:17:30,561 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T05:17:30,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9230 sec 2024-11-20T05:17:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T05:17:30,788 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:30,789 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T05:17:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:17:30,792 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:17:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T05:17:30,793 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:17:30,798 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,802 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/recovered.edits] 2024-11-20T05:17:30,806 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a85aca44d9274dba9a6e02f28d3a609d to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/a85aca44d9274dba9a6e02f28d3a609d 2024-11-20T05:17:30,808 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/c38a4295376d49069faac53c4cee76c8 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/A/c38a4295376d49069faac53c4cee76c8 2024-11-20T05:17:30,811 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a1fc90d77a5240ccbfd0c8045c88e9f1 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/a1fc90d77a5240ccbfd0c8045c88e9f1 2024-11-20T05:17:30,812 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de574a6b10f242b68ec83c284afe965a to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/B/de574a6b10f242b68ec83c284afe965a 2024-11-20T05:17:30,816 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/813b7b5e6d454f4d93228db5df98c44b to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/813b7b5e6d454f4d93228db5df98c44b 2024-11-20T05:17:30,818 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e5fecc96e0d640568d06a5eed372e063 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/C/e5fecc96e0d640568d06a5eed372e063 2024-11-20T05:17:30,823 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/recovered.edits/957.seqid to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4/recovered.edits/957.seqid 2024-11-20T05:17:30,824 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/default/TestAcidGuarantees/ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,824 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T05:17:30,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T05:17:30,826 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T05:17:30,831 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000ab3f118bb542a180dd67ba31057be8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000ab3f118bb542a180dd67ba31057be8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,832 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200aecedd433e84b5b9b63e00d945c5dee_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200aecedd433e84b5b9b63e00d945c5dee_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,833 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200feb17577d8d425b8f44472c336ec74b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200feb17577d8d425b8f44472c336ec74b_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,834 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112012e135fcd0a8485e9eb4cc82adf1a356_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112012e135fcd0a8485e9eb4cc82adf1a356_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,835 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112019de7e60d3a249baba421d5ee505e9a1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112019de7e60d3a249baba421d5ee505e9a1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,836 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203855960d0a284fe69fad72dfe4bcb8da_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203855960d0a284fe69fad72dfe4bcb8da_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,837 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039948d93088e4c97963551fad0ee5562_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039948d93088e4c97963551fad0ee5562_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,838 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203c2d08d663c6466b8245fa3af5216353_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203c2d08d663c6466b8245fa3af5216353_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,839 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112053d28e0e90ae4b7ebecdc4c4e33a1e95_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112053d28e0e90ae4b7ebecdc4c4e33a1e95_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,840 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206141645ada9240e4af1ae14a769177d1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206141645ada9240e4af1ae14a769177d1_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,840 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112063c78b4a2d5a4c8faee74ffba84c2c56_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112063c78b4a2d5a4c8faee74ffba84c2c56_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,841 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112068c49d24a52a47a6a081bb783034e87a_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112068c49d24a52a47a6a081bb783034e87a_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,842 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206ded9f44367a44e8b157bd8924e00cfc_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206ded9f44367a44e8b157bd8924e00cfc_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,842 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112096939bcc156b4d03b4389c4c9a65cdf3_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112096939bcc156b4d03b4389c4c9a65cdf3_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,843 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c4433cdbc6ea4fffa6137542cac36670_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c4433cdbc6ea4fffa6137542cac36670_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,844 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb1edd834d8f4b03ab4d36b4337715b8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb1edd834d8f4b03ab4d36b4337715b8_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,844 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d34229d3f40e4d55b9fefc796ff8f581_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d34229d3f40e4d55b9fefc796ff8f581_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,846 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d6645703ad6649918d73339315f97afb_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d6645703ad6649918d73339315f97afb_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,846 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de4afcb1b4734d49a1a0b85e2e8981c9_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de4afcb1b4734d49a1a0b85e2e8981c9_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,847 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e0da5f46f61f40488495b236e06b25c2_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 to hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e0da5f46f61f40488495b236e06b25c2_ee51cd6a06f90dcb79cbe1ee3eb2fbd4 2024-11-20T05:17:30,848 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T05:17:30,850 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:17:30,853 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T05:17:30,855 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T05:17:30,856 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:17:30,856 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T05:17:30,856 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732079850856"}]},"ts":"9223372036854775807"} 2024-11-20T05:17:30,858 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T05:17:30,858 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee51cd6a06f90dcb79cbe1ee3eb2fbd4, NAME => 'TestAcidGuarantees,,1732079815673.ee51cd6a06f90dcb79cbe1ee3eb2fbd4.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T05:17:30,858 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T05:17:30,858 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732079850858"}]},"ts":"9223372036854775807"} 2024-11-20T05:17:30,861 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T05:17:30,861 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T05:17:30,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-11-20T05:17:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T05:17:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T05:17:30,899 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T05:17:30,914 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 251), OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=382 (was 421), ProcessCount=11 (was 11), AvailableMemoryMB=5125 (was 4552) - AvailableMemoryMB LEAK? - 2024-11-20T05:17:30,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T05:17:30,914 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:17:30,915 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:17:30,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:30,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:30,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T05:17:30,915 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T05:17:30,915 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-20T05:17:30,915 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=51d5d2011a1c,37927,1732079654666 2024-11-20T05:17:30,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T05:17:30,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T05:17:30,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:17:30,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:17:30,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T05:17:30,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T05:17:30,979 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T05:17:30,979 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-20T05:17:30,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T05:17:30,979 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:17:30,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:30,979 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51d5d2011a1c,44753,1732079655401' ***** 2024-11-20T05:17:30,979 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T05:17:30,980 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T05:17:30,980 INFO [RS:0;51d5d2011a1c:44753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T05:17:30,980 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(959): stopping server 51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;51d5d2011a1c:44753. 2024-11-20T05:17:30,981 DEBUG [RS:0;51d5d2011a1c:44753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T05:17:30,981 DEBUG [RS:0;51d5d2011a1c:44753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T05:17:30,981 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T05:17:30,982 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T05:17:30,982 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T05:17:30,982 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T05:17:30,982 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T05:17:30,982 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T05:17:30,982 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T05:17:30,982 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T05:17:30,982 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T05:17:30,982 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T05:17:31,012 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/info/c8004bcb1d0f413d96802e802894ec10 is 95, key is TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b./info:/1732079693207/DeleteFamily/seqid=0 2024-11-20T05:17:31,014 INFO [regionserver/51d5d2011a1c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T05:17:31,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742432_1608 (size=6514) 2024-11-20T05:17:31,016 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/info/c8004bcb1d0f413d96802e802894ec10 2024-11-20T05:17:31,036 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/ns/1b9e09d4bd7545df9ee1b1fd823cd65d is 93, key is TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b./ns:/1732079693163/DeleteFamily/seqid=0 2024-11-20T05:17:31,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742433_1609 (size=5906) 2024-11-20T05:17:31,182 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T05:17:31,383 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T05:17:31,440 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/ns/1b9e09d4bd7545df9ee1b1fd823cd65d 2024-11-20T05:17:31,466 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/rep_barrier/9d4a031ee2bb4324a2cf1a62a477da94 is 102, key is TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b./rep_barrier:/1732079693163/DeleteFamily/seqid=0 2024-11-20T05:17:31,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742434_1610 (size=6025) 2024-11-20T05:17:31,583 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T05:17:31,614 INFO [regionserver/51d5d2011a1c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T05:17:31,614 INFO [regionserver/51d5d2011a1c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T05:17:31,783 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T05:17:31,872 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/rep_barrier/9d4a031ee2bb4324a2cf1a62a477da94 2024-11-20T05:17:31,897 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/table/2a3b588c57004825b55fc0a748cedfc9 is 96, key is TestAcidGuarantees,,1732079658693.098be5f00fb47f2a54b5983aaea0749b./table:/1732079693163/DeleteFamily/seqid=0 2024-11-20T05:17:31,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742435_1611 (size=5936) 2024-11-20T05:17:31,983 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T05:17:32,184 DEBUG [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T05:17:32,301 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/table/2a3b588c57004825b55fc0a748cedfc9 2024-11-20T05:17:32,304 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/info/c8004bcb1d0f413d96802e802894ec10 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/info/c8004bcb1d0f413d96802e802894ec10 2024-11-20T05:17:32,308 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/info/c8004bcb1d0f413d96802e802894ec10, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T05:17:32,309 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/ns/1b9e09d4bd7545df9ee1b1fd823cd65d as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/ns/1b9e09d4bd7545df9ee1b1fd823cd65d 2024-11-20T05:17:32,311 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/ns/1b9e09d4bd7545df9ee1b1fd823cd65d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T05:17:32,312 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/rep_barrier/9d4a031ee2bb4324a2cf1a62a477da94 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/rep_barrier/9d4a031ee2bb4324a2cf1a62a477da94 2024-11-20T05:17:32,315 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/rep_barrier/9d4a031ee2bb4324a2cf1a62a477da94, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T05:17:32,315 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/.tmp/table/2a3b588c57004825b55fc0a748cedfc9 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/table/2a3b588c57004825b55fc0a748cedfc9 2024-11-20T05:17:32,318 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/table/2a3b588c57004825b55fc0a748cedfc9, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T05:17:32,319 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1337ms, sequenceid=90, compaction requested=false 2024-11-20T05:17:32,323 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T05:17:32,324 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T05:17:32,324 INFO [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T05:17:32,324 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732079850982Running coprocessor pre-close hooks at 1732079850982Disabling compacts and flushes for region at 1732079850982Disabling writes for close at 1732079850982Obtaining lock to block concurrent updates at 1732079850982Preparing flush snapshotting stores in 1588230740 at 1732079850982Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732079850983 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732079850983Flushing 1588230740/info: creating writer at 1732079850984 (+1 ms)Flushing 1588230740/info: appending metadata at 1732079851012 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732079851012Flushing 1588230740/ns: creating writer at 1732079851021 (+9 ms)Flushing 1588230740/ns: appending metadata at 1732079851035 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732079851036 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1732079851447 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732079851466 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732079851466Flushing 1588230740/table: creating writer at 1732079851879 (+413 ms)Flushing 1588230740/table: appending metadata at 1732079851897 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732079851897Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a52bd6: reopening flushed file at 1732079852304 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3213499f: reopening flushed file at 1732079852308 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4ff596: reopening flushed file at 1732079852311 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2595e53f: reopening flushed file at 1732079852315 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1337ms, sequenceid=90, compaction requested=false at 1732079852319 (+4 ms)Writing region close event to WAL at 1732079852320 (+1 ms)Running coprocessor post-close hooks at 1732079852323 (+3 ms)Closed at 1732079852324 (+1 ms) 2024-11-20T05:17:32,324 DEBUG [RS_CLOSE_META-regionserver/51d5d2011a1c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T05:17:32,384 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(976): stopping server 51d5d2011a1c,44753,1732079655401; all regions closed. 2024-11-20T05:17:32,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741834_1010 (size=25344) 2024-11-20T05:17:32,397 DEBUG [RS:0;51d5d2011a1c:44753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/oldWALs 2024-11-20T05:17:32,397 INFO [RS:0;51d5d2011a1c:44753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51d5d2011a1c%2C44753%2C1732079655401.meta:.meta(num 1732079658273) 2024-11-20T05:17:32,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741833_1009 (size=20520099) 2024-11-20T05:17:32,399 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/WALs/51d5d2011a1c,44753,1732079655401/51d5d2011a1c%2C44753%2C1732079655401.1732079657767 not finished, retry = 0 2024-11-20T05:17:32,502 DEBUG [RS:0;51d5d2011a1c:44753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/oldWALs 2024-11-20T05:17:32,503 INFO [RS:0;51d5d2011a1c:44753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51d5d2011a1c%2C44753%2C1732079655401:(num 1732079657767) 2024-11-20T05:17:32,503 DEBUG [RS:0;51d5d2011a1c:44753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T05:17:32,503 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T05:17:32,503 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T05:17:32,503 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.ChoreService(370): Chore service for: regionserver/51d5d2011a1c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T05:17:32,503 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T05:17:32,504 INFO [regionserver/51d5d2011a1c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T05:17:32,504 INFO [RS:0;51d5d2011a1c:44753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44753 2024-11-20T05:17:32,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51d5d2011a1c,44753,1732079655401 2024-11-20T05:17:32,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T05:17:32,557 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T05:17:32,557 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51d5d2011a1c,44753,1732079655401] 2024-11-20T05:17:32,578 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51d5d2011a1c,44753,1732079655401 already deleted, retry=false 2024-11-20T05:17:32,578 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51d5d2011a1c,44753,1732079655401 expired; onlineServers=0 2024-11-20T05:17:32,578 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '51d5d2011a1c,37927,1732079654666' ***** 2024-11-20T05:17:32,578 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T05:17:32,578 INFO [M:0;51d5d2011a1c:37927 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T05:17:32,578 INFO [M:0;51d5d2011a1c:37927 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T05:17:32,578 DEBUG [M:0;51d5d2011a1c:37927 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T05:17:32,579 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T05:17:32,579 DEBUG [M:0;51d5d2011a1c:37927 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T05:17:32,579 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster-HFileCleaner.large.0-1732079657384 {}] cleaner.HFileCleaner(306): Exit Thread[master/51d5d2011a1c:0:becomeActiveMaster-HFileCleaner.large.0-1732079657384,5,FailOnTimeoutGroup] 2024-11-20T05:17:32,579 DEBUG [master/51d5d2011a1c:0:becomeActiveMaster-HFileCleaner.small.0-1732079657391 {}] cleaner.HFileCleaner(306): Exit Thread[master/51d5d2011a1c:0:becomeActiveMaster-HFileCleaner.small.0-1732079657391,5,FailOnTimeoutGroup] 2024-11-20T05:17:32,579 INFO [M:0;51d5d2011a1c:37927 {}] hbase.ChoreService(370): Chore service for: master/51d5d2011a1c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T05:17:32,580 INFO [M:0;51d5d2011a1c:37927 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T05:17:32,580 DEBUG [M:0;51d5d2011a1c:37927 {}] master.HMaster(1795): Stopping service threads 2024-11-20T05:17:32,580 INFO [M:0;51d5d2011a1c:37927 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T05:17:32,580 INFO [M:0;51d5d2011a1c:37927 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T05:17:32,580 ERROR [M:0;51d5d2011a1c:37927 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1420519831) connection to localhost/127.0.0.1:39657 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39657,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T05:17:32,581 INFO [M:0;51d5d2011a1c:37927 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T05:17:32,581 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T05:17:32,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T05:17:32,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T05:17:32,588 DEBUG [M:0;51d5d2011a1c:37927 {}] zookeeper.ZKUtil(347): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T05:17:32,589 WARN [M:0;51d5d2011a1c:37927 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T05:17:32,589 INFO [M:0;51d5d2011a1c:37927 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/.lastflushedseqids 2024-11-20T05:17:32,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742436_1612 (size=72) 2024-11-20T05:17:32,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T05:17:32,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44753-0x10156aa0f7c0001, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T05:17:32,668 INFO [RS:0;51d5d2011a1c:44753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T05:17:32,668 INFO [RS:0;51d5d2011a1c:44753 {}] regionserver.HRegionServer(1031): Exiting; stopping=51d5d2011a1c,44753,1732079655401; zookeeper connection closed. 2024-11-20T05:17:32,669 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e076ff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e076ff 2024-11-20T05:17:32,670 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T05:17:33,003 INFO [M:0;51d5d2011a1c:37927 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T05:17:33,003 INFO [M:0;51d5d2011a1c:37927 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T05:17:33,003 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T05:17:33,003 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:17:33,003 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:17:33,003 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T05:17:33,003 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:17:33,003 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=749.81 KB heapSize=916.84 KB 2024-11-20T05:17:33,023 DEBUG [M:0;51d5d2011a1c:37927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfcd3d52a68b431aaf14181a7e847d23 is 82, key is hbase:meta,,1/info:regioninfo/1732079658352/Put/seqid=0 2024-11-20T05:17:33,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742437_1613 (size=5672) 2024-11-20T05:17:33,430 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2061 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfcd3d52a68b431aaf14181a7e847d23 2024-11-20T05:17:33,460 DEBUG [M:0;51d5d2011a1c:37927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08f8b9548895428691aae97f70648bc7 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x87/proc:d/1732079818774/Put/seqid=0 2024-11-20T05:17:33,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742438_1614 (size=43676) 2024-11-20T05:17:33,864 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=749.26 KB at sequenceid=2061 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08f8b9548895428691aae97f70648bc7 2024-11-20T05:17:33,867 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 08f8b9548895428691aae97f70648bc7 2024-11-20T05:17:33,880 DEBUG [M:0;51d5d2011a1c:37927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4c6d249e4fc445eebfef507e1bd9e613 is 69, key is 51d5d2011a1c,44753,1732079655401/rs:state/1732079657531/Put/seqid=0 2024-11-20T05:17:33,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742439_1615 (size=5156) 2024-11-20T05:17:34,285 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2061 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4c6d249e4fc445eebfef507e1bd9e613 2024-11-20T05:17:34,297 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfcd3d52a68b431aaf14181a7e847d23 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dfcd3d52a68b431aaf14181a7e847d23 2024-11-20T05:17:34,301 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dfcd3d52a68b431aaf14181a7e847d23, entries=8, sequenceid=2061, filesize=5.5 K 2024-11-20T05:17:34,302 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08f8b9548895428691aae97f70648bc7 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/08f8b9548895428691aae97f70648bc7 2024-11-20T05:17:34,304 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 08f8b9548895428691aae97f70648bc7 2024-11-20T05:17:34,304 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/08f8b9548895428691aae97f70648bc7, entries=160, sequenceid=2061, filesize=42.7 K 2024-11-20T05:17:34,305 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4c6d249e4fc445eebfef507e1bd9e613 as hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4c6d249e4fc445eebfef507e1bd9e613 2024-11-20T05:17:34,307 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/7167e6b9-095b-5a00-fc42-114ec23ef673/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4c6d249e4fc445eebfef507e1bd9e613, entries=1, sequenceid=2061, filesize=5.0 K 2024-11-20T05:17:34,308 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(3140): Finished flush of dataSize ~749.81 KB/767809, heapSize ~916.54 KB/938536, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1305ms, sequenceid=2061, compaction requested=false 2024-11-20T05:17:34,309 INFO [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T05:17:34,309 DEBUG [M:0;51d5d2011a1c:37927 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732079853003Disabling compacts and flushes for region at 1732079853003Disabling writes for close at 1732079853003Obtaining lock to block concurrent updates at 1732079853003Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732079853003Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=767809, getHeapSize=938776, getOffHeapSize=0, getCellsCount=2246 at 1732079853004 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732079853004Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732079853005 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732079853023 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732079853023Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732079853440 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732079853460 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732079853460Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732079853867 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732079853880 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732079853880Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@270b83d2: reopening flushed file at 1732079854295 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61756ad7: reopening flushed file at 1732079854301 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c62a529: reopening flushed file at 1732079854304 (+3 ms)Finished flush of dataSize ~749.81 KB/767809, heapSize ~916.54 KB/938536, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1305ms, sequenceid=2061, compaction requested=false at 1732079854308 (+4 ms)Writing region close event to WAL at 1732079854309 (+1 ms)Closed at 1732079854309 2024-11-20T05:17:34,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741830_1006 (size=902933) 2024-11-20T05:17:34,311 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T05:17:34,312 INFO [M:0;51d5d2011a1c:37927 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T05:17:34,312 INFO [M:0;51d5d2011a1c:37927 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37927 2024-11-20T05:17:34,312 INFO [M:0;51d5d2011a1c:37927 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T05:17:34,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T05:17:34,489 INFO [M:0;51d5d2011a1c:37927 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T05:17:34,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37927-0x10156aa0f7c0000, quorum=127.0.0.1:60391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T05:17:34,498 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T05:17:34,500 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T05:17:34,500 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T05:17:34,500 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T05:17:34,500 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/hadoop.log.dir/,STOPPED} 2024-11-20T05:17:34,502 WARN [BP-1659978148-172.17.0.3-1732079650746 heartbeating to localhost/127.0.0.1:39657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T05:17:34,502 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T05:17:34,502 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T05:17:34,502 WARN [BP-1659978148-172.17.0.3-1732079650746 heartbeating to localhost/127.0.0.1:39657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1659978148-172.17.0.3-1732079650746 (Datanode Uuid 534a6e25-a7f9-42cd-a80a-545dcd92dfad) service to localhost/127.0.0.1:39657 2024-11-20T05:17:34,506 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/data/data1/current/BP-1659978148-172.17.0.3-1732079650746 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T05:17:34,506 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/cluster_588fbdf7-46e0-f040-4ce5-e4c2d94deba1/data/data2/current/BP-1659978148-172.17.0.3-1732079650746 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T05:17:34,507 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T05:17:34,515 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T05:17:34,516 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T05:17:34,516 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T05:17:34,516 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T05:17:34,516 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/836f8f0c-8d94-c177-8238-5be96b238aca/hadoop.log.dir/,STOPPED} 2024-11-20T05:17:34,533 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T05:17:34,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down